Rebase from fork

.ci/java-versions.properties

@@ -8,3 +8,4 @@ ES_BUILD_JAVA=openjdk12
 ES_RUNTIME_JAVA=java11
 GRADLE_TASK=build
 
+

.ci/matrix-runtime-javas.yml

@@ -9,6 +9,7 @@ ES_RUNTIME_JAVA:
  - java11
  - java12
  - openjdk12
+ - openjdk13
  - zulu11
  - zulu12
  - corretto11

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,5 +10,5 @@ attention.
 - Have you followed the [contributor guidelines](https:/elastic/elasticsearch/blob/master/CONTRIBUTING.md)?
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
 - If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
-- If submitting code, have you checked that your submission is for an [OS that we support](https://www.elastic.co/support/matrix#show_os)?
+- If submitting code, have you checked that your submission is for an [OS and architecture that we support](https://www.elastic.co/support/matrix#show_os)?
 - If you are submitting this code for a class then read our [policy](https:/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.

CONTRIBUTING.md

@@ -325,8 +325,7 @@ Commercially licensed code that integrates with the rest of Elasticsearch. The
 `docs` subdirectory functions just like the top level `docs` subdirectory and
 the `qa` subdirectory functions just like the top level `qa` subdirectory. The
 `plugin` subdirectory contains the x-pack module which runs inside the
-Elasticsearch process. The `transport-client` subdirectory contains extensions
-to Elasticsearch's standard transport client to work properly with x-pack.
+Elasticsearch process.
 
 ### Gradle Build
 

README.textile

@@ -150,13 +150,13 @@ curl -XPUT 'http://localhost:9200/kimchy/_doc/2?pretty' -H 'Content-Type: applic
 
 The above will index information into the @kimchy@ index. Each user will get their own special index.
 
-Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
+Complete control on the index level is allowed. As an example, in the above case, we might want to change from the default 1 shard with 1 replica per index, to 2 shards with 1 replica per index (because this user tweets a lot). Here is how this can be done (the configuration can be in yaml as well):
 
 <pre>
 curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
 {
  "index" : {
- "number_of_shards" : 1,
+ "number_of_shards" : 2,
  "number_of_replicas" : 1
  }
 }'
@@ -191,7 +191,7 @@ h3. Distributed, Highly Available
 
 Let's face it, things will fail....
 
-Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
+Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 1 shards and 1 replica per shard (1/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
 
 In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.
 

TESTING.asciidoc

@@ -305,12 +305,6 @@ e.g. -Dtests.rest.suite=index,get,create/10_with_id
 blacklisted and need to be skipped
 e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
 
-Note that the REST tests, like all the integration tests, can be run against an external
-cluster by specifying the `tests.cluster` property, which if present needs to contain a
-comma separated list of nodes to connect to (e.g. localhost:9300). A transport client will
-be created based on that and used for all the before|after test operations, and to extract
-the http addresses of the nodes so that REST requests can be sent to them.
-
 == Testing packaging
 
 The packaging tests use Vagrant virtual machines to verify that installing

benchmarks/build.gradle

@@ -27,7 +27,7 @@ archivesBaseName = 'elasticsearch-benchmarks'
 test.enabled = false
 
 dependencies {
- compile("org.elasticsearch:elasticsearch:${version}") {
+ compile(project(":server")) {
  // JMH ships with the conflicting version 4.6. This prevents us from using jopt-simple in benchmarks (which should be ok) but allows
  // us to invoke the JMH uberjar as usual.
  exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
@@ -92,7 +93,7 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl
 
  public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
  return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS,
- portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
- DiscoveryNode.Role.DATA), Version.CURRENT);
+ portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE,
+ DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
  }
 }

build.gradle

@@ -104,10 +104,7 @@ subprojects {
 
 /* Introspect all versions of ES that may be tested against for backwards
  * compatibility. It is *super* important that this logic is the same as the
- * logic in VersionUtils.java, throwing out alphas because they don't have any
- * backwards compatibility guarantees and only keeping the latest beta or rc
- * in a branch if there are only betas and rcs in the branch so we have
- * *something* to test against. */
+ * logic in VersionUtils.java. */
 BwcVersions versions = new BwcVersions(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 
 // build metadata from previous build, contains eg hashes for bwc builds
@@ -209,68 +206,7 @@ allprojects {
  javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
  }
 
- /* Sets up the dependencies that we build as part of this project but
- register as though they were external to resolve internally. We register
- them as external dependencies so the build plugin that we use can be used
- to build elasticsearch plugins outside of the elasticsearch source tree. */
- ext.projectSubstitutions = [
- "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
- "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
- "org.elasticsearch:elasticsearch:${version}": ':server',
- "org.elasticsearch:elasticsearch-cli:${version}": ':libs:elasticsearch-cli',
- "org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
- "org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
- "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
- "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo',
- "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
- "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config',
- "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
- "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
- "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
- "org.elasticsearch.client:test:${version}": ':client:test',
- "org.elasticsearch.client:transport:${version}": ':client:transport',
- "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
- "org.elasticsearch.test:framework:${version}": ':test:framework',
- "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
- "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
- // for transport client
- "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
- "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
- "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
- "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
- "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
- "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
- "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
- // for security example plugins
- "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core'
- ]
-
- /*
- * Gradle only resolve project substitutions during dependency resolution but
- * we sometimes want to do the resolution at other times. This creates a
- * convenient method we can call to do it.
- */
- ext.dependencyToProject = { Dependency dep ->
- if (dep instanceof ProjectDependency) {
- return dep.dependencyProject
- } else {
- String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
- if (substitution != null) {
- return findProject(substitution)
- }
- return null
- }
- }
-
  project.afterEvaluate {
- configurations.matching { it.canBeResolved }.all {
- resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
- projectSubstitutions.each { k,v ->
- subs.substitute(subs.module(k)).with(subs.project(v))
- }
- }
- }
-
  // Handle javadoc dependencies across projects. Order matters: the linksOffline for
  // org.elasticsearch:elasticsearch must be the last one or all the links for the
  // other packages (e.g org.elasticsearch.client) will point to server rather than
@@ -279,10 +215,10 @@ allprojects {
  String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
  Closure sortClosure = { a, b -> b.group <=> a.group }
  Closure depJavadocClosure = { shadowed, dep ->
- if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
+ if ((dep instanceof ProjectDependency) == false) {
  return
  }
- Project upstreamProject = project.ext.dependencyToProject(dep)
+ Project upstreamProject = dep.dependencyProject
  if (upstreamProject == null) {
  return
  }
@@ -338,8 +274,8 @@ gradle.projectsEvaluated {
  integTest.mustRunAfter test
  }
  configurations.matching { it.canBeResolved }.all { Configuration configuration ->
- dependencies.all { Dependency dep ->
- Project upstreamProject = dependencyToProject(dep)
+ dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
+ Project upstreamProject = dep.dependencyProject
  if (upstreamProject != null) {
  if (project.path == upstreamProject.path) {
  // TODO: distribution integ tests depend on themselves (!), fix that
@@ -552,31 +488,6 @@ gradle.projectsEvaluated {
  }
 }
 
-if (System.properties.get("build.compare") != null) {
- apply plugin: 'compare-gradle-builds'
- compareGradleBuilds {
- ext.referenceProject = System.properties.get("build.compare")
- doFirst {
- if (file(referenceProject).exists() == false) {
- throw new GradleException(
- "Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
- )
- }
- }
- sourceBuild {
- gradleVersion = gradle.getGradleVersion()
- projectDir = referenceProject
- tasks = ["clean", "assemble"]
- arguments = ["-Dbuild.compare_friendly=true"]
- }
- targetBuild {
- tasks = ["clean", "assemble"]
- // use -Dorg.gradle.java.home= to alter jdk versions
- arguments = ["-Dbuild.compare_friendly=true"]
- }
- }
-}
-
 allprojects {
  task resolveAllDependencies {
  dependsOn tasks.matching { it.name == "pullFixture"}

buildSrc/build.gradle

@@ -67,8 +67,26 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
 }
 
 // Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
-targetCompatibility = '8'
-sourceCompatibility = '8'
+targetCompatibility = '11'
+sourceCompatibility = '11'
+
+sourceSets {
+ // We have a few classes that need to be compiled for older java versions
+ minimumRuntime { }
+}
+
+compileMinimumRuntimeJava {
+ targetCompatibility = 8
+ sourceCompatibility = 8
+}
+
+jar {
+ from sourceSets.minimumRuntime.output
+}
+
+javadoc {
+ source sourceSets.minimumRuntime.allSource
+}
 
 /*****************************************************************************
  * Dependencies used by the entire build *
@@ -79,8 +97,15 @@ repositories {
 }
 
 dependencies {
+ if (project.ext.has("isEclipse") == false || project.ext.isEclipse == false) {
+ // eclipse is confused if this is set explicitly
+ compile sourceSets.minimumRuntime.output
+ }
+
  compile localGroovy()
 
+ compile 'commons-codec:commons-codec:1.12'
+
  compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
  compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
@@ -94,6 +119,9 @@ dependencies {
  testCompile "junit:junit:${props.getProperty('junit')}"
  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
  testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+ minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
+ minimumRuntimeCompile localGroovy()
+ minimumRuntimeCompile gradleApi()
 }
 
 /*****************************************************************************
@@ -121,17 +149,14 @@ if (project != rootProject) {
  apply plugin: 'nebula.maven-base-publish'
  apply plugin: 'nebula.maven-scm'
 
- // we need to apply these again to override the build plugin
- targetCompatibility = "10"
- sourceCompatibility = "10"
-
  // groovydoc succeeds, but has some weird internal exception...
  groovydoc.enabled = false
 
  // build-tools is not ready for primetime with these...
  dependencyLicenses.enabled = false
  dependenciesInfo.enabled = false
  forbiddenApisMain.enabled = false
+ forbiddenApisMinimumRuntime.enabled = false
  forbiddenApisTest.enabled = false
  jarHell.enabled = false
  thirdPartyAudit.enabled = false
@@ -148,6 +173,11 @@ if (project != rootProject) {
  distribution project(':distribution:archives:linux-tar')
  distribution project(':distribution:archives:oss-linux-tar')
  }
+
+ // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
+ processResources {
+ exclude 'buildSrc.marker'
+ }
 
  String localDownloads = "${rootProject.buildDir}/local-downloads"
  task setupLocalDownloads(type:Copy) {
@@ -178,6 +208,8 @@ if (project != rootProject) {
  systemProperty 'test.lucene-snapshot-revision', isLuceneSnapshot[0][1]
  }
  maxParallelForks System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel.toString()) as Integer
+ // These tests run Gradle which doesn't have FIPS support
+ onlyIf { project.inFipsJvm == false }
  }
  check.dependsOn(integTest)