Merge branch '6.x' into ccr-6.x

* 6.x:
  Upgrade to Lucene-7.4.0-snapshot-518d303506 (#31360)
  [ML] Implement new rules design (#31110) (#31294)
  Remove RestGetAllAliasesAction (#31308)
  CCS: don't proxy requests for already connected node (#31273)
  Rankeval: Fold template test project into main module (#31203)
  [Docs] Remove reference to repository-s3 plugin creating an S3 bucket (#31359)
  More detailed tracing when writing metadata (#31319)
  Add details section for dcg ranking metric (#31177)

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch = 6.4.0
-lucene = 7.4.0-snapshot-0a7c3f462f
+lucene = 7.4.0-snapshot-518d303506
 
 # optional dependencies
 spatial4j = 0.7

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -608,7 +609,7 @@ public void testDefaultNamedXContents() {
 
  public void testProvidedNamedXContents() {
  List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
- assertEquals(7, namedXContents.size());
+ assertEquals(8, namedXContents.size());
  Map<Class<?>, Integer> categories = new HashMap<>();
  List<String> names = new ArrayList<>();
  for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -626,9 +627,10 @@ public void testProvidedNamedXContents() {
  assertTrue(names.contains(PrecisionAtK.NAME));
  assertTrue(names.contains(DiscountedCumulativeGain.NAME));
  assertTrue(names.contains(MeanReciprocalRank.NAME));
- assertEquals(Integer.valueOf(2), categories.get(MetricDetail.class));
+ assertEquals(Integer.valueOf(3), categories.get(MetricDetail.class));
  assertTrue(names.contains(PrecisionAtK.NAME));
  assertTrue(names.contains(MeanReciprocalRank.NAME));
+ assertTrue(names.contains(DiscountedCumulativeGain.NAME));
  }
 
  private static class TrackingActionListener implements ActionListener<Integer> {

docs/plugins/repository-s3.asciidoc

@@ -284,22 +284,7 @@ You may further restrict the permissions by specifying a prefix within the bucke
 // NOTCONSOLE
 
 The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository
-registration will fail. If you want Elasticsearch to create the bucket instead, you can add the permission to create a
-specific bucket like this:
-
-[source,js]
-----
-{
- "Action": [
- "s3:CreateBucket"
- ],
- "Effect": "Allow",
- "Resource": [
- "arn:aws:s3:::snaps.example.com"
- ]
-}
-----
-// NOTCONSOLE
+registration will fail.
 
 [[repository-s3-aws-vpc]]
 [float]

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+a57659a275921d8ab3f7ec580e9bf713ce6143b1

modules/rank-eval/build.gradle

@@ -23,3 +23,12 @@ esplugin {
  hasClientJar = true
 }
 
+integTestCluster {
+ // Modules who's integration is explicitly tested in integration tests
+ module project(':modules:lang-mustache')
+}
+
+run {
+ // Modules who's integration is explicitly tested in integration tests
+ module project(':modules:lang-mustache')
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java

@@ -36,6 +36,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
 
@@ -129,26 +130,31 @@ public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
  .collect(Collectors.toList());
  List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
  List<Integer> ratingsInSearchHits = new ArrayList<>(ratedHits.size());
+ int unratedResults = 0;
  for (RatedSearchHit hit : ratedHits) {
- // unknownDocRating might be null, which means it will be unrated docs are
- // ignored in the dcg calculation
- // we still need to add them as a placeholder so the rank of the subsequent
- // ratings is correct
+ // unknownDocRating might be null, in which case unrated docs will be ignored in the dcg calculation.
+ // we still need to add them as a placeholder so the rank of the subsequent ratings is correct
  ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating));
+ if (hit.getRating().isPresent() == false) {
+ unratedResults++;
+ }
  }
- double dcg = computeDCG(ratingsInSearchHits);
+ final double dcg = computeDCG(ratingsInSearchHits);
+ double result = dcg;
+ double idcg = 0;
 
  if (normalize) {
  Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder()));
- double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
- if (idcg > 0) {
- dcg = dcg / idcg;
+ idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
+ if (idcg != 0) {
+ result = dcg / idcg;
  } else {
- dcg = 0;
+ result = 0;
  }
  }
- EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg);
+ EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, result);
  evalQueryQuality.addHitsAndRatings(ratedHits);
+ evalQueryQuality.setMetricDetails(new Detail(dcg, idcg, unratedResults));
  return evalQueryQuality;
  }
 
@@ -167,7 +173,7 @@ private static double computeDCG(List<Integer> ratings) {
  private static final ParseField K_FIELD = new ParseField("k");
  private static final ParseField NORMALIZE_FIELD = new ParseField("normalize");
  private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating");
- private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg_at", false,
+ private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg", false,
  args -> {
  Boolean normalized = (Boolean) args[0];
  Integer optK = (Integer) args[2];
@@ -217,4 +223,118 @@ public final boolean equals(Object obj) {
  public final int hashCode() {
  return Objects.hash(normalize, unknownDocRating, k);
  }
+
+ public static final class Detail implements MetricDetail {
+
+ private static ParseField DCG_FIELD = new ParseField("dcg");
+ private static ParseField IDCG_FIELD = new ParseField("ideal_dcg");
+ private static ParseField NDCG_FIELD = new ParseField("normalized_dcg");
+ private static ParseField UNRATED_FIELD = new ParseField("unrated_docs");
+ private final double dcg;
+ private final double idcg;
+ private final int unratedDocs;
+
+ Detail(double dcg, double idcg, int unratedDocs) {
+ this.dcg = dcg;
+ this.idcg = idcg;
+ this.unratedDocs = unratedDocs;
+ }
+
+ Detail(StreamInput in) throws IOException {
+ this.dcg = in.readDouble();
+ this.idcg = in.readDouble();
+ this.unratedDocs = in.readVInt();
+ }
+
+ @Override
+ public
+ String getMetricName() {
+ return NAME;
+ }
+
+ @Override
+ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(DCG_FIELD.getPreferredName(), this.dcg);
+ if (this.idcg != 0) {
+ builder.field(IDCG_FIELD.getPreferredName(), this.idcg);
+ builder.field(NDCG_FIELD.getPreferredName(), this.dcg / this.idcg);
+ }
+ builder.field(UNRATED_FIELD.getPreferredName(), this.unratedDocs);
+ return builder;
+ }
+
+ private static final ConstructingObjectParser<Detail, Void> PARSER = new ConstructingObjectParser<>(NAME, true, args -> {
+ return new Detail((Double) args[0], (Double) args[1] != null ? (Double) args[1] : 0.0d, (Integer) args[2]);
+ });
+
+ static {
+ PARSER.declareDouble(constructorArg(), DCG_FIELD);
+ PARSER.declareDouble(optionalConstructorArg(), IDCG_FIELD);
+ PARSER.declareInt(constructorArg(), UNRATED_FIELD);
+ }
+
+ public static Detail fromXContent(XContentParser parser) {
+ return PARSER.apply(parser, null);
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeDouble(this.dcg);
+ out.writeDouble(this.idcg);
+ out.writeVInt(this.unratedDocs);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ /**
+ * @return the discounted cumulative gain
+ */
+ public double getDCG() {
+ return this.dcg;
+ }
+
+ /**
+ * @return the ideal discounted cumulative gain, can be 0 if nothing was computed, e.g. because no normalization was required
+ */
+ public double getIDCG() {
+ return this.idcg;
+ }
+
+ /**
+ * @return the normalized discounted cumulative gain, can be 0 if nothing was computed, e.g. because no normalization was required
+ */
+ public double getNDCG() {
+ return (this.idcg != 0) ? this.dcg / this.idcg : 0;
+ }
+
+ /**
+ * @return the number of unrated documents in the search results
+ */
+ public Object getUnratedDocs() {
+ return this.unratedDocs;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ DiscountedCumulativeGain.Detail other = (DiscountedCumulativeGain.Detail) obj;
+ return (this.dcg == other.dcg &&
+ this.idcg == other.idcg &&
+ this.unratedDocs == other.unratedDocs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(this.dcg, this.idcg, this.unratedDocs);
+ }
+ }
 }
+

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java

@@ -41,6 +41,8 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
  PrecisionAtK.Detail::fromXContent));
  namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(MeanReciprocalRank.NAME),
  MeanReciprocalRank.Detail::fromXContent));
+ namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(DiscountedCumulativeGain.NAME),
+ DiscountedCumulativeGain.Detail::fromXContent));
  return namedXContent;
  }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java

@@ -61,8 +61,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
  namedWriteables.add(
  new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
  namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, PrecisionAtK.NAME, PrecisionAtK.Detail::new));
- namedWriteables
- .add(new NamedWriteableRegistry.Entry(MetricDetail.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Detail::new));
+ namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Detail::new));
+ namedWriteables.add(
+ new NamedWriteableRegistry.Entry(MetricDetail.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain.Detail::new));
  return namedWriteables;
  }
 

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.text.Text;
@@ -254,9 +255,8 @@ private void assertParsedCorrect(String xContent, Integer expectedUnknownDocRati
 
  public static DiscountedCumulativeGain createTestItem() {
  boolean normalize = randomBoolean();
- Integer unknownDocRating = Integer.valueOf(randomIntBetween(0, 1000));
-
- return new DiscountedCumulativeGain(normalize, unknownDocRating, 10);
+ Integer unknownDocRating = frequently() ? Integer.valueOf(randomIntBetween(0, 1000)) : null;
+ return new DiscountedCumulativeGain(normalize, unknownDocRating, randomIntBetween(1, 10));
  }
 
  public void testXContentRoundtrip() throws IOException {
@@ -283,7 +283,25 @@ public void testXContentParsingIsNotLenient() throws IOException {
  parser.nextToken();
  XContentParseException exception = expectThrows(XContentParseException.class,
  () -> DiscountedCumulativeGain.fromXContent(parser));
- assertThat(exception.getMessage(), containsString("[dcg_at] unknown field"));
+ assertThat(exception.getMessage(), containsString("[dcg] unknown field"));
+ }
+ }
+
+ public void testMetricDetails() {
+ double dcg = randomDoubleBetween(0, 1, true);
+ double idcg = randomBoolean() ? 0.0 : randomDoubleBetween(0, 1, true);
+ double expectedNdcg = idcg != 0 ? dcg / idcg : 0.0;
+ int unratedDocs = randomIntBetween(0, 100);
+ DiscountedCumulativeGain.Detail detail = new DiscountedCumulativeGain.Detail(dcg, idcg, unratedDocs);
+ assertEquals(dcg, detail.getDCG(), 0.0);
+ assertEquals(idcg, detail.getIDCG(), 0.0);
+ assertEquals(expectedNdcg, detail.getNDCG(), 0.0);
+ assertEquals(unratedDocs, detail.getUnratedDocs());
+ if (idcg != 0) {
+ assertEquals("{\"dcg\":{\"dcg\":" + dcg + ",\"ideal_dcg\":" + idcg + ",\"normalized_dcg\":" + expectedNdcg
+ + ",\"unrated_docs\":" + unratedDocs + "}}", Strings.toString(detail));
+ } else {
+ assertEquals("{\"dcg\":{\"dcg\":" + dcg + ",\"unrated_docs\":" + unratedDocs + "}}", Strings.toString(detail));
  }
  }
 

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java

@@ -68,10 +68,20 @@ public static EvalQueryQuality randomEvalQueryQuality() {
  EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10),
  randomDoubleBetween(0.0, 1.0, true));
  if (randomBoolean()) {
- if (randomBoolean()) {
+ int metricDetail = randomIntBetween(0, 2);
+ switch (metricDetail) {
+ case 0:
  evalQueryQuality.setMetricDetails(new PrecisionAtK.Detail(randomIntBetween(0, 1000), randomIntBetween(0, 1000)));
- } else {
+ break;
+ case 1:
  evalQueryQuality.setMetricDetails(new MeanReciprocalRank.Detail(randomIntBetween(0, 1000)));
+ break;
+ case 2:
+ evalQueryQuality.setMetricDetails(new DiscountedCumulativeGain.Detail(randomDoubleBetween(0, 1, true),
+ randomBoolean() ? randomDoubleBetween(0, 1, true) : 0, randomInt()));
+ break;
+ default:
+ throw new IllegalArgumentException("illegal randomized value in test");
  }
  }
  evalQueryQuality.addHitsAndRatings(ratedHits);

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+b91a260d8d12ee4b3302a63059c73a34de0ce146

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+cc1ca9bd9e2c162dd1da8c2e7111913fd8033e48

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+2fa3662a10a9e085b1c7b87293d727422cbe6224

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+60aa50c11857e6739e68936cb45102562b2c46b4

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+4586368007785a3be26db4b9ce404ffb8c76f350

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+9c6d030ab2c148df7a6ba73a774ef4b8c720a6cb

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+8275bf8df2644d5fcec2963cf237d14b6e00fefe