diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java index c6f33aa15db1d..4223931c4e32c 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java @@ -53,9 +53,9 @@ public void testNoData() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Collections.singletonList("field")); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); assertNull(stats.getStats()); - assertFalse(MatrixAggregationInspectionHelper.hasValue(stats)); + assertEquals(0L, stats.getDocCount()); } } } @@ -72,9 +72,9 @@ public void testUnmapped() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Collections.singletonList("bogus")); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); assertNull(stats.getStats()); - assertFalse(MatrixAggregationInspectionHelper.hasValue(stats)); + assertEquals(0L, stats.getDocCount()); } } } @@ -85,43 +85,6 @@ public void testTwoFields() throws Exception { String fieldB = "b"; MappedFieldType ftB = new NumberFieldMapper.NumberFieldType(fieldB, NumberFieldMapper.NumberType.DOUBLE); - try (Directory directory = newDirectory(); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { - - int numDocs = scaledRandomIntBetween(8192, 16384); - Double[] fieldAValues = new Double[numDocs]; - Double[] fieldBValues = new Double[numDocs]; - for (int docId = 0; docId < numDocs; docId++) { - Document document = new Document(); - fieldAValues[docId] = randomDouble(); - document.add(new SortedNumericDocValuesField(fieldA, NumericUtils.doubleToSortableLong(fieldAValues[docId]))); - - fieldBValues[docId] = randomDouble(); - document.add(new SortedNumericDocValuesField(fieldB, NumericUtils.doubleToSortableLong(fieldBValues[docId]))); - indexWriter.addDocument(document); - } - - MultiPassStats multiPassStats = new MultiPassStats(fieldA, fieldB); - multiPassStats.computeStats(Arrays.asList(fieldAValues), Arrays.asList(fieldBValues)); - try (IndexReader reader = indexWriter.getReader()) { - IndexSearcher searcher = new IndexSearcher(reader); - MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") - .fields(Arrays.asList(fieldA, fieldB)); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ftA, ftB); - // Since `search` doesn't do any reduction, and the InternalMatrixStats object will have a null `MatrixStatsResults` - // object. That is created during the final reduction, which also does a final round of computations - // So we have to create a MatrixStatsResults object here manually so that the final `compute()` is called - multiPassStats.assertNearlyEqual(new MatrixStatsResults(stats.getStats())); - } - } - } - - public void testTwoFieldsReduce() throws Exception { - String fieldA = "a"; - MappedFieldType ftA = new NumberFieldMapper.NumberFieldType(fieldA, NumberFieldMapper.NumberType.DOUBLE); - String fieldB = "b"; - MappedFieldType ftB = new NumberFieldMapper.NumberFieldType(fieldB, NumberFieldMapper.NumberType.DOUBLE); - try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { @@ -145,8 +108,6 @@ public void testTwoFieldsReduce() throws Exception { MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Arrays.asList(fieldA, fieldB)); InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ftA, ftB); - // Unlike testTwoFields, `searchAndReduce` will execute reductions so the `MatrixStatsResults` object - // will be populated and fully computed. We should use that value directly to test against multiPassStats.assertNearlyEqual(stats); assertTrue(MatrixAggregationInspectionHelper.hasValue(stats)); } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index 3a4a263a84b76..af4a0e0bc9530 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -303,7 +303,7 @@ private void testCase(Query query, IndexSearcher indexSearcher, Consumer buckets, List plan, } toMerge.add(buckets.get(startIdx)); // Don't remove the startIdx bucket because it will be replaced by the merged bucket - reduceContext.consumeBucketsAndMaybeBreak(- (toMerge.size() - 1)); + int toRemove = toMerge.stream().mapToInt(b -> countInnerBucket(b)+1).sum(); + reduceContext.consumeBucketsAndMaybeBreak(-toRemove + 1); Bucket merged_bucket = reduceBucket(toMerge, reduceContext); buckets.set(startIdx, merged_bucket); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index a2e6a16dbb7f8..4f81a955b9e28 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1879,7 +1879,7 @@ public void testEarlyTermination() throws Exception { ) ); - executeTestCase(true, false, new TermQuery(new Term("foo", "bar")), + executeTestCase(true, new TermQuery(new Term("foo", "bar")), dataset, () -> new CompositeAggregationBuilder("name", @@ -1899,7 +1899,7 @@ public void testEarlyTermination() throws Exception { ); // source field and index sorting config have different order - executeTestCase(true, false, new TermQuery(new Term("foo", "bar")), + executeTestCase(true, new TermQuery(new Term("foo", "bar")), dataset, () -> new CompositeAggregationBuilder("name", @@ -1936,7 +1936,7 @@ public void testIndexSortWithDuplicate() throws Exception { ); for (SortOrder order : SortOrder.values()) { - executeTestCase(true, false, new MatchAllDocsQuery(), + executeTestCase(true, new MatchAllDocsQuery(), dataset, () -> new CompositeAggregationBuilder("name", @@ -1959,7 +1959,7 @@ public void testIndexSortWithDuplicate() throws Exception { } ); - executeTestCase(true, false, new MatchAllDocsQuery(), + executeTestCase(true, new MatchAllDocsQuery(), dataset, () -> new CompositeAggregationBuilder("name", @@ -1989,14 +1989,12 @@ private void testSearchCase(List queries, Supplier create, Consumer verify) throws IOException { for (Query query : queries) { - executeTestCase(false, false, query, dataset, create, verify); - executeTestCase(false, true, query, dataset, create, verify); - executeTestCase(true, true, query, dataset, create, verify); + executeTestCase(false, query, dataset, create, verify); + executeTestCase(true, query, dataset, create, verify); } } private void executeTestCase(boolean useIndexSort, - boolean reduced, Query query, List>> dataset, Supplier create, @@ -2019,18 +2017,13 @@ private void executeTestCase(boolean useIndexSort, indexWriter.addDocument(document); document.clear(); } - if (reduced == false && randomBoolean()) { + if (rarely()) { indexWriter.forceMerge(1); } } try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); - final InternalComposite composite; - if (reduced) { - composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - } else { - composite = search(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - } + InternalComposite composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); verify.accept(composite); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java index 5afad0400149a..e1b1e951df470 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java @@ -62,7 +62,7 @@ public void testEmpty() throws Exception { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); QueryBuilder filter = QueryBuilders.termQuery("field", randomAlphaOfLength(5)); FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter); - InternalFilter response = search(indexSearcher, new MatchAllDocsQuery(), builder, + InternalFilter response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(response.getDocCount(), 0); assertFalse(AggregationInspectionHelper.hasValue(response)); @@ -80,7 +80,7 @@ public void testRandom() throws Exception { for (int i = 0; i < numDocs; i++) { if (frequently()) { // make sure we have more than one segment to test the merge - indexWriter.getReader().close(); + indexWriter.commit(); } int value = randomInt(maxTerm-1); expectedBucketCount[value] += 1; @@ -98,20 +98,12 @@ public void testRandom() throws Exception { QueryBuilder filter = QueryBuilders.termQuery("field", Integer.toString(value)); FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter); - for (boolean doReduce : new boolean[]{true, false}) { - final InternalFilter response; - if (doReduce) { - response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, - fieldType); - } else { - response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - assertEquals(response.getDocCount(), (long) expectedBucketCount[value]); - if (expectedBucketCount[value] > 0) { - assertTrue(AggregationInspectionHelper.hasValue(response)); - } else { - assertFalse(AggregationInspectionHelper.hasValue(response)); - } + final InternalFilter response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + assertEquals(response.getDocCount(), (long) expectedBucketCount[value]); + if (expectedBucketCount[value] > 0) { + assertTrue(AggregationInspectionHelper.hasValue(response)); + } else { + assertFalse(AggregationInspectionHelper.hasValue(response)); } } finally { indexReader.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 265115e720600..617eec9799a4d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -60,7 +60,7 @@ public void testEmpty() throws Exception { } FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters); builder.otherBucketKey("other"); - InternalFilters response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + InternalFilters response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(response.getBuckets().size(), numFilters); for (InternalFilters.InternalBucket filter : response.getBuckets()) { assertEquals(filter.getDocCount(), 0); @@ -113,22 +113,15 @@ public void testKeyedFilter() throws Exception { FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", keys); builder.otherBucket(true); builder.otherBucketKey("other"); - for (boolean doReduce : new boolean[] {true, false}) { - final InternalFilters filters; - if (doReduce) { - filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } else { - filters = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - assertEquals(filters.getBuckets().size(), 7); - assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2); - assertEquals(filters.getBucketByKey("foo").getDocCount(), 2); - assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2); - assertEquals(filters.getBucketByKey("bar").getDocCount(), 1); - assertEquals(filters.getBucketByKey("same").getDocCount(), 1); - assertEquals(filters.getBucketByKey("other").getDocCount(), 2); - assertTrue(AggregationInspectionHelper.hasValue(filters)); - } + final InternalFilters filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + assertEquals(filters.getBuckets().size(), 7); + assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2); + assertEquals(filters.getBucketByKey("foo").getDocCount(), 2); + assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2); + assertEquals(filters.getBucketByKey("bar").getDocCount(), 1); + assertEquals(filters.getBucketByKey("same").getDocCount(), 1); + assertEquals(filters.getBucketByKey("other").getDocCount(), 2); + assertTrue(AggregationInspectionHelper.hasValue(filters)); indexReader.close(); directory.close(); @@ -175,28 +168,21 @@ public void testRandom() throws Exception { builder.otherBucket(true); builder.otherBucketKey("other"); - for (boolean doReduce : new boolean[]{true, false}) { - final InternalFilters response; - if (doReduce) { - response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } else { - response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - List buckets = response.getBuckets(); - assertEquals(buckets.size(), filters.length + 1); + final InternalFilters response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + List buckets = response.getBuckets(); + assertEquals(buckets.size(), filters.length + 1); - for (InternalFilters.InternalBucket bucket : buckets) { - if ("other".equals(bucket.getKey())) { - assertEquals(bucket.getDocCount(), expectedOtherCount); - } else { - int index = Integer.parseInt(bucket.getKey()); - assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]); - } + for (InternalFilters.InternalBucket bucket : buckets) { + if ("other".equals(bucket.getKey())) { + assertEquals(bucket.getDocCount(), expectedOtherCount); + } else { + int index = Integer.parseInt(bucket.getKey()); + assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]); } - - // Always true because we include 'other' in the agg - assertTrue(AggregationInspectionHelper.hasValue(response)); } + + // Always true because we include 'other' in the agg + assertTrue(AggregationInspectionHelper.hasValue(response)); } finally { indexReader.close(); directory.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index 477df9e59163c..4cb34acab2449 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -98,7 +98,7 @@ public class AutoDateHistogramAggregatorTests extends DateHistogramAggregatorTes private static final Query DEFAULT_QUERY = new MatchAllDocsQuery(); public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATES_WITH_TIME, + testSearchCase(new MatchNoDocsQuery(), DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -115,20 +115,16 @@ public void testMatchAllDocs() throws IOException { expectedDocCount.put("2015-01-01T00:00:00.000Z", 3); expectedDocCount.put("2016-01-01T00:00:00.000Z", 1); expectedDocCount.put("2017-01-01T00:00:00.000Z", 1); - testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2011-01-01T00:00:00.000Z", 0); expectedDocCount.put("2014-01-01T00:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); } public void testSubAggregations() throws IOException { - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD) .subAggregation(AggregationBuilders.stats("stats").field(DATE_FIELD)), histogram -> { @@ -249,7 +245,7 @@ public void testAsSubAgg() throws IOException { expectedMax.put("2020-01-01T00:00:00.000Z", 2.0); expectedMax.put("2021-01-01T00:00:00.000Z", 3.0); assertThat(maxAsMap(ak1adh), equalTo(expectedMax)); - + StringTerms.Bucket b = terms.getBucketByKey("b"); StringTerms bk1 = b.getAggregations().get("k1"); StringTerms.Bucket bk1a = bk1.getBucketByKey("a"); @@ -391,7 +387,7 @@ public void testNoDocs() throws IOException { assertFalse(AggregationInspectionHelper.hasValue(histogram)); } ); - testSearchAndReduceCase(DEFAULT_QUERY, dates, aggregation, + testSearchCase(DEFAULT_QUERY, dates, aggregation, histogram -> { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); @@ -431,7 +427,7 @@ public void testIntervalYear() throws IOException { final long start = LocalDate.of(2015, 1, 1).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); final long end = LocalDate.of(2017, 12, 31).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); final Query rangeQuery = LongPoint.newRangeQuery(INSTANT_FIELD, start, end); - testBothCases(rangeQuery, DATES_WITH_TIME, + testSearchCase(rangeQuery, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), histogram -> { final ZonedDateTime startDate = ZonedDateTime.of(2015, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); @@ -460,7 +456,7 @@ public void testIntervalMonth() throws IOException { expectedDocCount.put("2017-01-01T00:00:00.000Z", 1); expectedDocCount.put("2017-02-01T00:00:00.000Z", 2); expectedDocCount.put("2017-03-01T00:00:00.000Z", 3); - testBothCases(DEFAULT_QUERY, datesForMonthInterval, + testSearchCase(DEFAULT_QUERY, datesForMonthInterval, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -490,12 +486,8 @@ public void testIntervalDay() throws IOException { expectedDocCount.put("2017-02-02T00:00:00.000Z", 2); expectedDocCount.put("2017-02-03T00:00:00.000Z", 3); expectedDocCount.put("2017-02-05T00:00:00.000Z", 1); - testSearchCase(DEFAULT_QUERY, datesForDayInterval, - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-04T00:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + testSearchCase(DEFAULT_QUERY, datesForDayInterval, aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -515,12 +507,8 @@ public void testIntervalDayWithTZ() throws IOException { expectedDocCount.put("2017-02-01T00:00:00.000-01:00", 2); expectedDocCount.put("2017-02-02T00:00:00.000-01:00", 3); expectedDocCount.put("2017-02-04T00:00:00.000-01:00", 1); - testSearchCase(DEFAULT_QUERY, datesForDayInterval, - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-03T00:00:00.000-01:00", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + testSearchCase(DEFAULT_QUERY, datesForDayInterval, aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -546,13 +534,9 @@ public void testIntervalHour() throws IOException { expectedDocCount.put("2017-02-01T15:00:00.000Z", 1); expectedDocCount.put("2017-02-01T15:00:00.000Z", 1); expectedDocCount.put("2017-02-01T16:00:00.000Z", 3); - testSearchCase(DEFAULT_QUERY, datesForHourInterval, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T11:00:00.000Z", 0); expectedDocCount.put("2017-02-01T12:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -560,7 +544,7 @@ public void testIntervalHour() throws IOException { expectedDocCount.put("2017-02-01T09:00:00.000Z", 3); expectedDocCount.put("2017-02-01T12:00:00.000Z", 3); expectedDocCount.put("2017-02-01T15:00:00.000Z", 4); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -585,13 +569,9 @@ public void testIntervalHourWithTZ() throws IOException { expectedDocCount.put("2017-02-01T13:00:00.000-01:00", 2); expectedDocCount.put("2017-02-01T14:00:00.000-01:00", 1); expectedDocCount.put("2017-02-01T15:00:00.000-01:00", 3); - testSearchCase(DEFAULT_QUERY, datesForHourInterval, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T10:00:00.000-01:00", 0); expectedDocCount.put("2017-02-01T11:00:00.000-01:00", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -612,7 +592,7 @@ public void testRandomSecondIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 30); bucketsToExpectedDocCountMap.put(3, 60); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -641,7 +621,7 @@ public void testRandomMinuteIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 30); bucketsToExpectedDocCountMap.put(3, 60); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -669,7 +649,7 @@ public void testRandomHourIntervals() throws IOException { bucketsToExpectedDocCountMap.put(12, 12); bucketsToExpectedDocCountMap.put(3, 24); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -693,7 +673,7 @@ public void testRandomDayIntervals() throws IOException { } final int randomChoice = randomIntBetween(1, 3); if (randomChoice == 1) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(length).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -704,7 +684,7 @@ public void testRandomDayIntervals() throws IOException { assertEquals(1, bucket.getDocCount()); }); } else if (randomChoice == 2) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(60).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -716,7 +696,7 @@ public void testRandomDayIntervals() throws IOException { assertEquals(expectedDocCount, bucket.getDocCount()); }); } else if (randomChoice == 3) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -742,7 +722,7 @@ public void testRandomMonthIntervals() throws IOException { bucketsToExpectedDocCountMap.put(30, 3); bucketsToExpectedDocCountMap.put(6, 12); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -772,7 +752,7 @@ public void testRandomYearIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 50); bucketsToExpectedDocCountMap.put(5, 100); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -797,29 +777,20 @@ public void testIntervalMinute() throws IOException { skeletonDocCount.put("2017-02-01T09:02:00.000Z", 2); skeletonDocCount.put("2017-02-01T09:15:00.000Z", 1); skeletonDocCount.put("2017-02-01T09:16:00.000Z", 2); - testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(skeletonDocCount)) - ); Map fullDocCount = new TreeMap<>(); fullDocCount.put("2017-02-01T09:02:00.000Z", 2); fullDocCount.put("2017-02-01T09:07:00.000Z", 0); fullDocCount.put("2017-02-01T09:12:00.000Z", 3); - testSearchAndReduceCase(DEFAULT_QUERY, datesForMinuteInterval, + testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(fullDocCount)) ); - - testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, - aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(skeletonDocCount)) - ); fullDocCount.clear(); fullDocCount.putAll(skeletonDocCount); for (int minute = 3; minute < 15; minute++) { - fullDocCount.put(String.format(Locale.ROOT, "2017-02-01T09:%02d:00.000Z", minute), 0); + fullDocCount.put(String.format(Locale.ROOT, "2017-02-01T09:%02d:00.000Z", minute), 0); } - testSearchAndReduceCase(DEFAULT_QUERY, datesForMinuteInterval, + testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(fullDocCount)) ); @@ -837,22 +808,18 @@ public void testIntervalSecond() throws IOException { expectedDocCount.put("2017-02-01T00:00:05.000Z", 1); expectedDocCount.put("2017-02-01T00:00:07.000Z", 2); expectedDocCount.put("2017-02-01T00:00:11.000Z", 3); - testSearchCase(DEFAULT_QUERY, datesForSecondInterval, - aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T00:00:06.000Z", 0); expectedDocCount.put("2017-02-01T00:00:08.000Z", 0); expectedDocCount.put("2017-02-01T00:00:09.000Z", 0); expectedDocCount.put("2017-02-01T00:00:10.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForSecondInterval, + testSearchCase(DEFAULT_QUERY, datesForSecondInterval, aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); } public void testWithPipelineReductions() throws IOException { - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(1).field(DATE_FIELD) .subAggregation(AggregationBuilders.histogram("histo").field(NUMERIC_FIELD).interval(1) .subAggregation(AggregationBuilders.max("max").field(NUMERIC_FIELD)) @@ -880,25 +847,6 @@ public void testWithPipelineReductions() throws IOException { }); } - private void testSearchCase(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - } - - private void testSearchAndReduceCase(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, configure, verify); - } - - private void testBothCases(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - executeTestCase(true, query, dataset, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { final Settings nodeSettings = Settings.builder() @@ -913,7 +861,7 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(final boolean reduced, final Query query, final List dataset, + private void testSearchCase(final Query query, final List dataset, final Consumer configure, final Consumer verify) throws IOException { try (Directory directory = newDirectory()) { @@ -936,12 +884,8 @@ private void executeTestCase(final boolean reduced, final Query query, final Lis MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NUMERIC_FIELD, NumberFieldMapper.NumberType.LONG); - final InternalAutoDateHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); - } + final InternalAutoDateHistogram histogram = + searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); verify.accept(histogram); } } @@ -951,10 +895,6 @@ private void indexSampleData(List dataset, RandomIndexWriter inde final Document document = new Document(); int i = 0; for (final ZonedDateTime date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - final long instant = date.toInstant().toEpochMilli(); document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); document.add(new LongPoint(INSTANT_FIELD, instant)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 4d84a854fc57f..d297d9fabc613 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -67,7 +67,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas "2017-12-12T22:55:46"); public void testMatchNoDocsDeprecatedInterval() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -78,11 +78,11 @@ public void testMatchNoDocsDeprecatedInterval() throws IOException { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -92,20 +92,13 @@ public void testMatchAllDocsDeprecatedInterval() throws IOException { Query query = new MatchAllDocsQuery(); testSearchCase(query, DATASET, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), - histogram -> { - assertEquals(6, histogram.getBuckets().size()); - assertTrue(AggregationInspectionHelper.hasValue(histogram)); - }, false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(8, histogram.getBuckets().size()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); }, false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { assertEquals(6, histogram.getBuckets().size()); @@ -122,34 +115,26 @@ public void testMatchAllDocs() throws IOException { for (int i = 0; i < 1000; i++) { foo.add(DATASET.get(randomIntBetween(0, DATASET.size()-1))); } - testSearchAndReduceCase(query, foo, + testSearchCase(query, foo, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")) .field(AGGREGABLE_DATE).order(BucketOrder.count(false)), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); testSearchCase(query, DATASET, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), - histogram -> assertEquals(6, histogram.getBuckets().size()), false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); testSearchCase(query, DATASET, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), - histogram -> assertEquals(6, histogram.getBuckets().size()), false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); @@ -206,7 +191,7 @@ public void testNoDocsDeprecatedInterval() throws IOException { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); }, false); - testSearchAndReduceCase(query, dates, aggregation, histogram -> { + testSearchCase(query, dates, aggregation, histogram -> { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); }, false); @@ -221,7 +206,7 @@ public void testNoDocs() throws IOException { testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testSearchAndReduceCase(query, dates, aggregation, + testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -230,13 +215,13 @@ public void testNoDocs() throws IOException { testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testSearchAndReduceCase(query, dates, aggregation, + testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); } public void testAggregateWrongFieldDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field("wrong_field"), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -247,18 +232,18 @@ public void testAggregateWrongFieldDeprecated() throws IOException { } public void testAggregateWrongField() throws IOException { - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field("wrong_field"), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field("wrong_field"), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); } public void testIntervalYearDeprecated() throws IOException { - testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, + testSearchCase(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); @@ -281,7 +266,7 @@ public void testIntervalYearDeprecated() throws IOException { } public void testIntervalYear() throws IOException { - testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, + testSearchCase(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); @@ -303,7 +288,7 @@ public void testIntervalYear() throws IOException { } public void testIntervalMonthDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { @@ -327,7 +312,7 @@ public void testIntervalMonthDeprecated() throws IOException { } public void testIntervalMonth() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), aggregation -> aggregation.calendarInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { @@ -350,7 +335,7 @@ public void testIntervalMonth() throws IOException { } public void testIntervalDayDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -386,7 +371,7 @@ public void testIntervalDayDeprecated() throws IOException { } public void testIntervalDay() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -418,7 +403,7 @@ public void testIntervalDay() throws IOException { assertEquals(1, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -453,7 +438,7 @@ public void testIntervalDay() throws IOException { } public void testIntervalHourDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -500,7 +485,7 @@ public void testIntervalHourDeprecated() throws IOException { } public void testIntervalHour() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -543,7 +528,7 @@ public void testIntervalHour() throws IOException { assertEquals(3, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -589,7 +574,7 @@ public void testIntervalHour() throws IOException { } public void testIntervalMinuteDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -619,7 +604,7 @@ public void testIntervalMinuteDeprecated() throws IOException { } public void testIntervalMinute() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -645,7 +630,7 @@ public void testIntervalMinute() throws IOException { assertEquals(2, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -674,7 +659,7 @@ public void testIntervalMinute() throws IOException { } public void testIntervalSecondDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -705,7 +690,7 @@ public void testIntervalSecondDeprecated() throws IOException { } public void testIntervalSecond() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -732,7 +717,7 @@ public void testIntervalSecond() throws IOException { assertEquals(3, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -762,7 +747,7 @@ public void testIntervalSecond() throws IOException { } public void testNanosIntervalSecond() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015298384Z", "2017-02-01T00:00:11.299954583Z", @@ -789,7 +774,7 @@ public void testNanosIntervalSecond() throws IOException { assertEquals(3, bucket.getDocCount()); }, true ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015298384Z", "2017-02-01T00:00:11.299954583Z", @@ -829,7 +814,7 @@ public void testMinDocCountDeprecated() throws IOException { ); // 5 sec interval with minDocCount = 0 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); @@ -854,7 +839,7 @@ public void testMinDocCountDeprecated() throws IOException { ); // 5 sec interval with minDocCount = 3 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); @@ -879,7 +864,7 @@ public void testMinDocCount() throws IOException { ); // 5 sec interval with minDocCount = 0 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); @@ -904,7 +889,7 @@ public void testMinDocCount() throws IOException { ); // 5 sec interval with minDocCount = 3 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); @@ -1175,43 +1160,6 @@ private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, int maxBucket, boolean useNanosecondResolution) throws IOException { - executeTestCase(false, query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, boolean useNanosecondResolution) throws IOException { - testSearchAndReduceCase(query, dataset, configure, verify, 1000, useNanosecondResolution); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - executeTestCase(true, query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, boolean useNanosecondResolution) throws IOException { - testBothCases(query, dataset, configure, verify, 10000, useNanosecondResolution); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - testSearchCase(query, dataset, configure, verify, maxBucket, useNanosecondResolution); - testSearchAndReduceCase(query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void executeTestCase(boolean reduced, - Query query, - List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - boolean aggregableDateIsSearchable = randomBoolean(); DateFieldMapper.DateFieldType fieldType = aggregableDateFieldType(useNanosecondResolution, aggregableDateIsSearchable); @@ -1220,10 +1168,6 @@ private void executeTestCase(boolean reduced, try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (String date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long instant = asLong(date, fieldType); document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant)); if (aggregableDateIsSearchable) { @@ -1243,12 +1187,7 @@ private void executeTestCase(boolean reduced, configure.accept(aggregationBuilder); } - InternalDateHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, maxBucket, null, fieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, maxBucket, fieldType); - } + InternalDateHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, maxBucket, fieldType); verify.accept(histogram); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java index b41aa2896d3be..81426688c3e94 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java @@ -55,7 +55,7 @@ public class NumericHistogramAggregatorTests extends AggregatorTestCase { public void testLongs() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (long value : new long[] {7, 3, -10, -6, 5, 50}) { + for (long value : new long[] {7, 3, -10, -6, 5, 15}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); @@ -66,16 +66,20 @@ public void testLongs() throws Exception { .interval(5); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -84,7 +88,7 @@ public void testLongs() throws Exception { public void testDoubles() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (double value : new double[] {9.3, 3.2, -10, -6.5, 5.3, 50.1}) { + for (double value : new double[] {9.3, 3.2, -10, -6.5, 5.3, 15.1}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))); w.addDocument(doc); @@ -95,16 +99,21 @@ public void testDoubles() throws Exception { .interval(5); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(6, histogram.getBuckets().size()); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -130,10 +139,6 @@ public void testDates() throws Exception { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir)) { Document document = new Document(); for (String date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long instant = fieldType.parse(date); document.add(new SortedNumericDocValuesField(fieldName, instant)); indexWriter.addDocument(document); @@ -145,7 +150,7 @@ public void testDates() throws Exception { .interval(1000 * 60 * 60 * 24); try (IndexReader reader = indexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -165,16 +170,20 @@ public void testIrrationalInterval() throws Exception { .interval(Math.PI); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-4 * Math.PI, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); assertEquals(-3 * Math.PI, histogram.getBuckets().get(1).getKey()); assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(Math.PI, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-2 * Math.PI, histogram.getBuckets().get(2).getKey()); + assertEquals(0, histogram.getBuckets().get(2).getDocCount()); + assertEquals(-Math.PI, histogram.getBuckets().get(3).getKey()); + assertEquals(0, histogram.getBuckets().get(3).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(4).getKey()); + assertEquals(2, histogram.getBuckets().get(4).getDocCount()); + assertEquals(Math.PI, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -209,7 +218,7 @@ public void testMinDocCount() throws Exception { public void testMissing() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (long value : new long[] {7, 3, -10, -6, 5, 50}) { + for (long value : new long[] {7, 3, -10, -6, 5, 15}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); @@ -222,16 +231,20 @@ public void testMissing() throws Exception { .missing(2d); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(7, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(7, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -252,7 +265,7 @@ public void testMissingUnmappedField() throws Exception { MappedFieldType type = null; try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, type); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, type); assertEquals(1, histogram.getBuckets().size()); @@ -280,7 +293,7 @@ public void testMissingUnmappedFieldBadType() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Throwable t = expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), aggBuilder, type); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, type); }); // This throws a number format exception (which is a subclass of IllegalArgumentException) and might be ok? assertThat(t.getMessage(), containsString(missingValue)); @@ -304,7 +317,7 @@ public void testIncorrectFieldType() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), aggBuilder, keywordField("field")); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, keywordField("field")); }); } } @@ -326,14 +339,16 @@ public void testOffset() throws Exception { .offset(Math.PI); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(3, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(4, histogram.getBuckets().size()); assertEquals(-10 + Math.PI, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(Math.PI, histogram.getBuckets().get(1).getKey()); - assertEquals(2, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5 + Math.PI, histogram.getBuckets().get(2).getKey()); - assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(-5 + Math.PI, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(Math.PI, histogram.getBuckets().get(2).getKey()); + assertEquals(2, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5 + Math.PI, histogram.getBuckets().get(3).getKey()); + assertEquals(1, histogram.getBuckets().get(3).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -358,18 +373,21 @@ public void testRandomOffset() throws Exception { .offset(offset); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(3, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(4, histogram.getBuckets().size()); assertEquals(-10 + expectedOffset, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); - assertEquals(expectedOffset, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); + assertEquals(-5 + expectedOffset, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5 + expectedOffset, histogram.getBuckets().get(2).getKey()); + assertEquals(expectedOffset, histogram.getBuckets().get(2).getKey()); assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5 + expectedOffset, histogram.getBuckets().get(3).getKey()); + assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java index 990c8840c3f9e..312ca43c7286e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java @@ -57,7 +57,7 @@ public void testDoubles() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -71,8 +71,9 @@ public void testDoubles() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -86,11 +87,14 @@ public void testDoubles() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -103,7 +107,7 @@ public void testLongs() throws Exception { new RangeFieldMapper.Range(rangeType, 1L, 5L, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3L, 4L, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4L, 13L, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42L, 49L, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22L, 29L, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -117,8 +121,9 @@ public void testLongs() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -132,11 +137,14 @@ public void testLongs() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -150,7 +158,7 @@ public void testMultipleRanges() throws Exception { new RangeFieldMapper.Range(rangeType, 1L, 5L, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3L, 4L, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4L, 13L, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42L, 49L, true, true) // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22L, 29L, true, true) // bucket 20, 25, 30 )); doc.add(new BinaryDocValuesField("field", encodedRange)); w.addDocument(doc); @@ -161,8 +169,9 @@ public void testMultipleRanges() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -176,11 +185,14 @@ public void testMultipleRanges() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } @@ -206,7 +218,8 @@ public void testMultipleRangesLotsOfOverlap() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); assertEquals(3, histogram.getBuckets().size()); assertEquals(0d, histogram.getBuckets().get(0).getKey()); @@ -243,7 +256,8 @@ public void testLongsIrrationalInterval() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); assertEquals(6, histogram.getBuckets().size()); assertEquals(-1 * Math.PI, histogram.getBuckets().get(0).getKey()); @@ -315,7 +329,7 @@ public void testOffset() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket -1, 4 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -6 -1 4 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 4, 9 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 39, 44, 49 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 19, 24, 29 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -330,8 +344,9 @@ public void testOffset() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - //assertEquals(7, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(8, histogram.getBuckets().size()); assertEquals(-6d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -345,14 +360,17 @@ public void testOffset() throws Exception { assertEquals(9d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(39d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(14d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(44d, histogram.getBuckets().get(5).getKey()); + assertEquals(19d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); - assertEquals(49d, histogram.getBuckets().get(6).getKey()); + assertEquals(24d, histogram.getBuckets().get(6).getKey()); assertEquals(1, histogram.getBuckets().get(6).getDocCount()); + + assertEquals(29d, histogram.getBuckets().get(7).getKey()); + assertEquals(1, histogram.getBuckets().get(7).getDocCount()); } } } @@ -365,7 +383,7 @@ public void testOffsetGtInterval() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -386,8 +404,9 @@ public void testOffsetGtInterval() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d + expectedOffset, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -401,11 +420,14 @@ public void testOffsetGtInterval() throws Exception { assertEquals(10d + expectedOffset, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d + expectedOffset, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d + expectedOffset, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d + expectedOffset, histogram.getBuckets().get(5).getKey()); + assertEquals(20d + expectedOffset, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d + expectedOffset, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -429,7 +451,7 @@ public void testIpRangesUnsupported() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Exception e = expectThrows(IllegalArgumentException.class, () -> - search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType))); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType))); assertThat(e.getMessage(), equalTo("Expected numeric range type but found non-numeric range [ip_range]")); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java index 8b24e92f6bb0e..c8fca52645a2c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java @@ -65,7 +65,7 @@ public class VariableWidthHistogramAggregatorTests extends AggregatorTestCase { public void testNoDocs() throws Exception{ final List dataset = Arrays.asList(); - testBothCases(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -87,7 +87,7 @@ public void testMoreClustersThanDocs() throws Exception { expectedMins.put(-3d, -3d); expectedMins.put(10d, 10d); - testBothCases(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -164,7 +164,7 @@ public void testDoubles() throws Exception { expectedMaxesOnlySearch.put(8.8, 8.8); testSearchCase(DEFAULT_QUERY, dataset, false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(6).setInitialBuffer(4), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedCentroidsOnlySearch.size(), buckets.size()); @@ -198,7 +198,7 @@ public void testDoubles() throws Exception { expectedMaxesSearchReduce.put(5.3, 5.9); expectedMaxesSearchReduce.put(8.8, 8.8); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, false, + testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -220,16 +220,12 @@ public void testNewBucketCreation() throws Exception { final List dataset = Arrays.asList(-1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 40, 30, 25, 32, 36, 80, 50, 75, 60); double doubleError = 1d / 10000d; - // Search (no reduce) - - // Expected clusters: [ (-1), (1), (3), (5), (7), (9), (11), (13), (15), (17), - // (19), (25, 30, 32), (36, 40, 50), (60), (75, 80) ] - // Corresponding keys (centroids): [ -1, 1, 3, ..., 17, 19, 29, 42, 77.5] - // Note: New buckets are created for 30, 50, and 80 because they are distant from the other buckets - final List keys = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 29d, 42d, 60d, 77.5d); - final List mins = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 25d, 36d, 60d, 75d); - final List maxes = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 32d, 50d, 60d, 80d); - final List docCounts = Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 2); + // Expected clusters: [ (-1, 1), (3, 5), (7, 9), (11, 13), (15, 17), + // (19), (25), (30), (32), (36), (40), (50), (60), (75), (80) ] + final List keys = Arrays.asList(0d, 4d, 8d, 12d, 16d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List mins = Arrays.asList(-1d, 3d, 7d, 11d, 15d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List maxes = Arrays.asList(1d, 5d, 9d, 13d, 17d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List docCounts = Arrays.asList(2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert keys.size() == docCounts.size() && keys.size() == keys.size(); final Map expectedDocCountOnlySearch = new HashMap<>(); @@ -242,7 +238,7 @@ public void testNewBucketCreation() throws Exception { } testSearchCase(DEFAULT_QUERY, dataset, false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(16).setInitialBuffer(12), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(15), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedDocCountOnlySearch.size(), buckets.size()); @@ -267,7 +263,7 @@ public void testNewBucketCreation() throws Exception { } testSearchCase(DEFAULT_QUERY, dataset.stream().map(n -> Double.valueOf(n.doubleValue() * Long.MAX_VALUE)).collect(toList()), false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(16).setInitialBuffer(12), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(15), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedDocCountOnlySearch.size(), buckets.size()); @@ -284,28 +280,22 @@ public void testNewBucketCreation() throws Exception { // There should not be more than `shard_size` documents on a node, even when very distant documents appear public void testNewBucketLimit() throws Exception{ - final List dataset = Arrays.asList(1,2,3,4,5, 10, 20, 50, 100, 5400, -900); + final List dataset = Arrays.asList(1, 2, 3, 4, 5, 10, 20, 50, 100, 5400, -900); double doubleError = 1d / 10000d; - // Expected clusters: [ (-900, 1, 2), (3, 4), (5), (10, 20, 50, 100, 5400)] - // Corresponding keys (centroids): [ -299, 3.5, 5, 1116] + // Expected clusters: [ (-900, 1, 2, 3, 4, 5), (10, 20, 50, 100, 5400)] + // Corresponding keys (centroids): [ -147.5, 1116] final Map expectedDocCount = new HashMap<>(); - expectedDocCount.put(-299d, 3); - expectedDocCount.put(3.5d, 2); - expectedDocCount.put(5d, 1); - expectedDocCount.put(1116d, 5); + expectedDocCount.put(-147.5d, 6); + expectedDocCount.put(1116.0d, 5); final Map expectedMins = new HashMap<>(); - expectedMins.put(-299d, -900d); - expectedMins.put(3.5d, 3d); - expectedMins.put(5d, 5d); - expectedMins.put(1116d, 10d); + expectedMins.put(-147.5d, -900d); + expectedMins.put(1116.0d, 10d); final Map expectedMaxes = new HashMap<>(); - expectedMaxes.put(-299d, 2d); - expectedMaxes.put(3.5d, 4d); - expectedMaxes.put(5d, 5d); - expectedMaxes.put(1116d, 5400d); + expectedMaxes.put(-147.5d, 5d); + expectedMaxes.put(1116.0d, 5400d); testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(2).setShardSize(4).setInitialBuffer(5), @@ -325,7 +315,7 @@ public void testNewBucketLimit() throws Exception{ public void testSimpleSubAggregations() throws IOException{ final List dataset = Arrays.asList(5, 1, 9, 2, 8); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, false, + testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(3) .setInitialBuffer(3) @@ -426,7 +416,7 @@ public void testMultipleSegments() throws IOException{ // To account for this case of a document switching clusters, we check that each cluster centroid is within // a certain range, rather than asserting exact values. - testSearchAndReduceCase(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(2) .setInitialBuffer(4) @@ -508,12 +498,11 @@ public void testSmallShardSize() throws Exception { public void testHugeShardSize() throws Exception { final List dataset = Arrays.asList(1, 2, 3); - testBothCases(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setShardSize(1000000000), histogram -> { - assertThat( + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setShardSize(1000000000), + histogram -> assertThat( histogram.getBuckets().stream().map(InternalVariableWidthHistogram.Bucket::getKey).collect(toList()), - equalTo(List.of(1.0, 2.0, 3.0)) - ); - }); + equalTo(List.of(1.0, 2.0, 3.0))) + ); } public void testSmallInitialBuffer() throws Exception { @@ -529,7 +518,7 @@ public void testSmallInitialBuffer() throws Exception { public void testOutOfOrderInitialBuffer() throws Exception { final List dataset = Arrays.asList(1, 2, 3); - testBothCases( + testSearchCase( DEFAULT_QUERY, dataset, true, @@ -553,26 +542,6 @@ public void testDefaultInitialBufferDependsOnNumBuckets() throws Exception { assertThat(new VariableWidthHistogramAggregationBuilder("test").setNumBuckets(3).getInitialBuffer(), equalTo(1500)); } - private void testSearchCase(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, multipleSegments, configure, verify); - } - - - private void testSearchAndReduceCase(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, multipleSegments, configure, verify); - } - - private void testBothCases(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, multipleSegments, configure, verify); - executeTestCase(false, query, dataset, multipleSegments, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { final Settings nodeSettings = Settings.builder() @@ -588,10 +557,9 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(final boolean reduced, final Query query, - final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { + private void testSearchCase(final Query query, final List dataset, boolean multipleSegments, + final Consumer configure, + final Consumer verify) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { indexSampleData(dataset, indexWriter, multipleSegments); @@ -619,12 +587,7 @@ private void executeTestCase(final boolean reduced, final Query query, - final InternalVariableWidthHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, fieldType); - } + final InternalVariableWidthHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); verify.accept(histogram); } } @@ -645,10 +608,6 @@ private void indexSampleData(List dataset, RandomIndexWriter indexWriter // Create multiple segments in the index final Document document = new Document(); for (final Number doc : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long fieldVal = convertDocumentToSortableValue(doc); document.add(new SortedNumericDocValuesField(NUMERIC_FIELD, fieldVal)); indexWriter.addDocument(document); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 517f1bd07c8ae..d437e18788610 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -393,17 +393,6 @@ private void testCase(Query query, CheckedConsumer writeIndex, Consumer verify, Collection fieldTypes) throws IOException { - testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false); - testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true); - } - - private void testCaseWithReduce(Query query, - MissingAggregationBuilder builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes, - boolean reduced) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { writeIndex.accept(indexWriter); @@ -412,12 +401,7 @@ private void testCaseWithReduce(Query query, try (IndexReader indexReader = DirectoryReader.open(directory)) { final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final InternalMissing missing; - if (reduced) { - missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); - } else { - missing = search(indexSearcher, query, builder, fieldTypesArray); - } + final InternalMissing missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); verify.accept(missing); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index fde8b7e2e3518..078450445f546 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -151,7 +151,7 @@ public void testNoDocs() throws IOException { nestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(NESTED_AGG, nested.getName()); @@ -196,7 +196,7 @@ public void testSingleNestingMax() throws IOException { nestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -248,7 +248,7 @@ public void testDoubleNestingMax() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -303,7 +303,7 @@ public void testOrphanedDocs() throws IOException { nestedBuilder.subAggregation(sumAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -386,7 +386,7 @@ public void testResetRootDocId() throws Exception { bq.add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term(IdFieldMapper.NAME, Uid.encodeId("2"))), BooleanClause.Occur.MUST_NOT); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new ConstantScoreQuery(bq.build()), nestedBuilder, fieldType); assertEquals(NESTED_AGG, nested.getName()); @@ -424,7 +424,7 @@ public void testNestedOrdering() throws IOException { nestedBuilder.subAggregation(maxAgg); termsBuilder.subAggregation(nestedBuilder); - Terms terms = search(newSearcher(indexReader, false, true), + Terms terms = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(7, terms.getBuckets().size()); @@ -473,7 +473,8 @@ public void testNestedOrdering() throws IOException { nestedBuilder.subAggregation(maxAgg); termsBuilder.subAggregation(nestedBuilder); - terms = search(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); + terms = searchAndReduce(newSearcher(indexReader, false, true), + new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(7, terms.getBuckets().size()); assertEquals("authors", terms.getName()); @@ -561,7 +562,7 @@ public void testNestedOrdering_random() throws IOException { nestedBuilder.subAggregation(minAgg); termsBuilder.subAggregation(nestedBuilder); - Terms terms = search(newSearcher(indexReader, false, true), + Terms terms = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(books.size(), terms.getBuckets().size()); @@ -658,7 +659,7 @@ public void testPreGetChildLeafCollectors() throws IOException { MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType("key"); MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType("value"); - Filter filter = search(newSearcher(indexReader, false, true), + Filter filter = searchAndReduce(newSearcher(indexReader, false, true), Queries.newNonNestedFilter(), filterAggregationBuilder, fieldType1, fieldType2); assertEquals("filterAgg", filter.getName()); @@ -719,9 +720,9 @@ public void testFieldAlias() throws IOException { NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation( max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias")); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), agg, fieldType); - Nested aliasNested = search(newSearcher(indexReader, false, true), + Nested aliasNested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), aliasAgg, fieldType); assertEquals(nested, aliasNested); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index 17e6c8e313f7f..dbcfa73d895b8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -98,7 +98,7 @@ public void testNoDocs() throws IOException { reverseNestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); ReverseNested reverseNested = (ReverseNested) ((InternalAggregation)nested).getProperty(REVERSE_AGG_NAME); @@ -160,7 +160,7 @@ public void testMaxFromParentDocs() throws IOException { reverseNestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -224,9 +224,9 @@ public void testFieldAlias() throws IOException { NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation( reverseNested(REVERSE_AGG_NAME).subAggregation(aliasMaxAgg)); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), agg, fieldType); - Nested aliasNested = search(newSearcher(indexReader, false, true), + Nested aliasNested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), aliasAgg, fieldType); ReverseNested reverseNested = nested.getAggregations().get(REVERSE_AGG_NAME); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java index f4dd09c6eebd7..b811e150fe407 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java @@ -113,7 +113,7 @@ public void testRanges() throws Exception { MappedFieldType fieldType = new IpFieldMapper.IpFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, fieldType); + InternalBinaryRange range = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(numRanges, range.getBuckets().size()); for (int i = 0; i < range.getBuckets().size(); i++) { Tuple expected = requestedRanges[i]; @@ -148,7 +148,7 @@ public void testMissingUnmapped() throws Exception { .missing("192.168.100.42"); // Apparently we expect a string here try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); + InternalBinaryRange range = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); assertEquals(1, range.getBuckets().size()); } } @@ -169,7 +169,7 @@ public void testMissingUnmappedBadType() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); + searchAndReduce(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); }); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java index f482342d25449..15ab2cc21a969 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java @@ -192,7 +192,7 @@ private void testCase(IndexSearcher indexSearcher, MappedFieldType genreFieldTyp .shardSize(shardSize) .subAggregation(new TermsAggregationBuilder("terms").field("id")); - InternalSampler result = search(indexSearcher, query, builder, genreFieldType, idFieldType); + InternalSampler result = searchAndReduce(indexSearcher, query, builder, genreFieldType, idFieldType); verify.accept(result); } @@ -211,7 +211,7 @@ public void testDiversifiedSampler_noDocs() throws Exception { .field(genreFieldType.name()) .subAggregation(new TermsAggregationBuilder("terms").field("id")); - InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType); + InternalSampler result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType); Terms terms = result.getAggregations().get("terms"); assertEquals(0, terms.getBuckets().size()); indexReader.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java index 8621475fa4c59..9f39e0e896d3a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java @@ -59,7 +59,7 @@ public class BinaryTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING ); @@ -68,7 +68,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -87,7 +87,7 @@ public void testBadIncludeExclude() throws IOException { // Make sure the include/exclude fails regardless of how the user tries to type hint the agg AggregationExecutionException e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), null // default, no hint )); @@ -95,7 +95,7 @@ public void testBadIncludeExclude() throws IOException { "they can only be applied to string fields. Use an array of values for include/exclude clauses")); e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), ValueType.STRING // string type hint )); @@ -104,7 +104,7 @@ public void testBadIncludeExclude() throws IOException { } public void testBadUserValueTypeHint() throws IOException { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testBothCases(new MatchNoDocsQuery(), dataset, + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> fail("test should have failed with exception"), ValueType.NUMERIC // numeric type hint )); @@ -114,34 +114,10 @@ public void testBadUserValueTypeHint() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (Long value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new BinaryFieldMapper.CustomBinaryDocValuesField(BINARY_FIELD, Numbers.longToBytes(value))); indexWriter.addDocument(document); document.clear(); @@ -161,12 +137,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType binaryFieldType = new BinaryFieldMapper.BinaryFieldType(BINARY_FIELD); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, binaryFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, binaryFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, binaryFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java index c3a1be37d05ba..4e049c7e3a176 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java @@ -56,12 +56,12 @@ public class KeywordTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING // with type hint ); @@ -70,7 +70,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -82,7 +82,7 @@ public void testMatchAllDocs() throws IOException { }, null // without type hint ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -98,34 +98,10 @@ public void testMatchAllDocs() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (String value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(value))); indexWriter.addDocument(document); document.clear(); @@ -145,12 +121,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType keywordFieldType = new KeywordFieldMapper.KeywordFieldType(KEYWORD_FIELD); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, keywordFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java index 30e32c7aa84bc..174a8a1aed6e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java @@ -59,12 +59,12 @@ public class NumericTermsAggregatorTests extends AggregatorTestCase { public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.NUMERIC // with type hint ); @@ -73,7 +73,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -85,7 +85,7 @@ public void testMatchAllDocs() throws IOException { }, null //without type hint ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -104,7 +104,7 @@ public void testBadIncludeExclude() throws IOException { // Numerics don't support any regex include/exclude, so should fail no matter what we do AggregationExecutionException e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), null )); @@ -113,7 +113,7 @@ public void testBadIncludeExclude() throws IOException { "values for include/exclude clauses used to filter numeric fields")); e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), ValueType.NUMERIC // with type hint )); @@ -126,34 +126,10 @@ public void testBadIncludeExclude() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (Long value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedNumericDocValuesField(LONG_FIELD, value)); document.add(new LongPoint(LONG_FIELD, value)); indexWriter.addDocument(document); @@ -175,12 +151,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType longFieldType = new NumberFieldMapper.NumberFieldType(LONG_FIELD, NumberFieldMapper.NumberType.LONG); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, longFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, longFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, longFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index 17a61818e7768..78e8ca6f68147 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -99,11 +99,11 @@ public class RareTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); @@ -112,7 +112,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -121,7 +121,7 @@ public void testMatchAllDocs() throws IOException { assertThat(bucket.getDocCount(), equalTo(1L)); } ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -144,7 +144,7 @@ public void testManyDocsOneRare() throws IOException { // The one rare term d.add(0L); - testSearchAndReduceCase(query, d, + testSearchCase(query, d, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -153,7 +153,7 @@ public void testManyDocsOneRare() throws IOException { assertThat(bucket.getDocCount(), equalTo(1L)); } ); - testSearchAndReduceCase(query, d, + testSearchCase(query, d, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -167,7 +167,7 @@ public void testManyDocsOneRare() throws IOException { public void testIncludeExclude() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD) .maxDocCount(2) // bump to 2 since we're only including "2" .includeExclude(new IncludeExclude(new long[]{2}, new long[]{})), @@ -178,7 +178,7 @@ public void testIncludeExclude() throws IOException { assertThat(bucket.getDocCount(), equalTo(2L)); } ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD) .maxDocCount(2) // bump to 2 since we're only including "2" .includeExclude(new IncludeExclude(new String[]{"2"}, new String[]{})), @@ -194,7 +194,7 @@ public void testIncludeExclude() throws IOException { public void testEmbeddedMaxAgg() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD); aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(max); }, @@ -210,7 +210,7 @@ public void testEmbeddedMaxAgg() throws IOException { assertThat(((Max)(children.asList().get(0))).getValue(), equalTo(1.0)); } ); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD); aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(max); }, @@ -240,11 +240,11 @@ public void testEmpty() throws IOException { agg -> assertEquals(0, agg.getBuckets().size()) ); - testSearchAndReduceCase(query, Collections.emptyList(), + testSearchCase(query, Collections.emptyList(), aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); - testSearchAndReduceCase(query, Collections.emptyList(), + testSearchCase(query, Collections.emptyList(), aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); @@ -312,7 +312,7 @@ public void testRangeField() throws Exception { public void testNestedTerms() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms") .field(KEYWORD_FIELD); aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(terms); @@ -331,7 +331,7 @@ public void testNestedTerms() throws IOException { } ); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms") .field(KEYWORD_FIELD); aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(terms); @@ -352,22 +352,20 @@ public void testNestedTerms() throws IOException { } public void testInsideTerms() throws IOException { - for (boolean reduce : new boolean[] {false, true}) { - for (String field : new String[] {KEYWORD_FIELD, LONG_FIELD}) { - AggregationBuilder builder = new TermsAggregationBuilder("terms").field("even_odd").subAggregation( - new RareTermsAggregationBuilder("rare").field(field).maxDocCount(2)); - StringTerms terms = (StringTerms) executeTestCase(reduce, new MatchAllDocsQuery(), dataset, builder); - - StringTerms.Bucket even = terms.getBucketByKey("even"); - InternalRareTerms evenRare = even.getAggregations().get("rare"); - assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("2")); - assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(2L)); - - StringTerms.Bucket odd = terms.getBucketByKey("odd"); - InternalRareTerms oddRare = odd.getAggregations().get("rare"); - assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("1")); - assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(1L)); - } + for (String field : new String[] {KEYWORD_FIELD, LONG_FIELD}) { + AggregationBuilder builder = new TermsAggregationBuilder("terms").field("even_odd").subAggregation( + new RareTermsAggregationBuilder("rare").field(field).maxDocCount(2)); + StringTerms terms = executeTestCase(new MatchAllDocsQuery(), dataset, builder); + + StringTerms.Bucket even = terms.getBucketByKey("even"); + InternalRareTerms evenRare = even.getAggregations().get("rare"); + assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("2")); + assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(2L)); + + StringTerms.Bucket odd = terms.getBucketByKey("odd"); + InternalRareTerms oddRare = odd.getAggregations().get("rare"); + assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("1")); + assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(1L)); } } @@ -530,34 +528,6 @@ private List generateDocsWithNested(String id, int value, int[] nested return documents; } - private void testSearchCase( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - } - - private void testSearchAndReduceCase( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - executeTestCase(true, query, dataset, configure, verify); - } - - private void testBothCases( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - testSearchCase(query, dataset, configure, verify); - testSearchAndReduceCase(query, dataset, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { Settings nodeSettings = Settings.builder() @@ -572,38 +542,27 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer> verify) throws IOException { + private void testSearchCase(Query query, + List dataset, + Consumer configure, + Consumer> verify) throws IOException { RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name"); if (configure != null) { configure.accept(aggregationBuilder); } - InternalMappedRareTerms result = (InternalMappedRareTerms) executeTestCase( - reduced, - query, - dataset, - aggregationBuilder - ); - verify.accept(result); + verify.accept(executeTestCase(query, dataset, aggregationBuilder)); + } - private InternalAggregation executeTestCase( - boolean reduced, - Query query, - List dataset, - AggregationBuilder aggregationBuilder - ) throws IOException { + private A executeTestCase(Query query, + List dataset, + AggregationBuilder aggregationBuilder) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); List shuffledDataset = new ArrayList<>(dataset); Collections.shuffle(shuffledDataset, random()); for (Long value : shuffledDataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedNumericDocValuesField(LONG_FIELD, value)); document.add(new LongPoint(LONG_FIELD, value)); document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(Long.toString(value)))); @@ -620,10 +579,7 @@ private InternalAggregation executeTestCase( keywordField(KEYWORD_FIELD), longField(LONG_FIELD), keywordField("even_odd")}; - if (reduced) { - return searchAndReduce(indexSearcher, query, aggregationBuilder, types); - } - return search(indexSearcher, query, aggregationBuilder, types); + return searchAndReduce(indexSearcher, query, aggregationBuilder, types); } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index 59bdda66dea41..e992eab92bf04 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -115,7 +115,7 @@ public void testSignificance() throws IOException { indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("text"); sigAgg.executionHint(randomExecutionHint()); @@ -259,7 +259,7 @@ public void testUnmapped() throws IOException { indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); // Attempt aggregation on unmapped field SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("unmapped_field"); @@ -328,7 +328,7 @@ public void testFieldAlias() throws IOException { indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); SignificantTermsAggregationBuilder agg = significantTerms("sig_text").field("text"); SignificantTermsAggregationBuilder aliasAgg = significantTerms("sig_text").field("text-alias"); @@ -389,7 +389,7 @@ private void testAllDocsWithoutStringField(String executionHint) throws IOExcept IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f") .executionHint(executionHint); - SignificantStringTerms result = search(searcher, new MatchAllDocsQuery(), request, keywordField("f")); + SignificantStringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("f")); assertThat(result.getSubsetSize(), equalTo(1L)); } } @@ -409,7 +409,7 @@ public void testAllDocsWithoutNumericField() throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f"); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, longField("f")); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("f")); assertThat(result.getSubsetSize(), equalTo(1L)); } } @@ -441,7 +441,7 @@ private void testSomeDocsWithoutStringField(String executionHint) throws IOExcep IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f") .executionHint(executionHint); - SignificantStringTerms result = search(searcher, new MatchAllDocsQuery(), request, keywordField("f")); + SignificantStringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("f")); assertThat(result.getSubsetSize(), equalTo(2L)); } } @@ -463,7 +463,7 @@ public void testSomeDocsWithoutNumericField() throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f"); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, longField("f")); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("f")); assertThat(result.getSubsetSize(), equalTo(2L)); } } @@ -495,14 +495,17 @@ private void threeLayerStringTestCase(String executionHint) throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder kRequest = new SignificantTermsAggregationBuilder("k").field("k") + .minDocCount(0) .executionHint(executionHint); SignificantTermsAggregationBuilder jRequest = new SignificantTermsAggregationBuilder("j").field("j") + .minDocCount(0) .executionHint(executionHint) .subAggregation(kRequest); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i") + .minDocCount(0) .executionHint(executionHint) .subAggregation(jRequest); - SignificantStringTerms result = search( + SignificantStringTerms result = searchAndReduce( searcher, new MatchAllDocsQuery(), request, @@ -549,10 +552,10 @@ public void testThreeLayerLong() throws IOException { } try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); - SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i") - .subAggregation(new SignificantTermsAggregationBuilder("j").field("j") - .subAggregation(new SignificantTermsAggregationBuilder("k").field("k"))); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, + SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i").minDocCount(0) + .subAggregation(new SignificantTermsAggregationBuilder("j").field("j").minDocCount(0) + .subAggregation(new SignificantTermsAggregationBuilder("k").field("k").minDocCount(0))); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("i"), longField("j"), longField("k")); assertThat(result.getSubsetSize(), equalTo(1000L)); for (int i = 0; i < 10; i++) { @@ -576,7 +579,7 @@ public void testThreeLayerLong() throws IOException { } } - private void addMixedTextDocs(TextFieldType textFieldType, IndexWriter w) throws IOException { + private void addMixedTextDocs(IndexWriter w) throws IOException { for (int i = 0; i < 10; i++) { Document doc = new Document(); StringBuilder text = new StringBuilder("common "); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index ad2304c58d9f5..45a7db50cc0f5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -1206,7 +1206,7 @@ public void testWithNestedAggregations() throws IOException { = new NumberFieldMapper.NumberFieldType("nested_value", NumberFieldMapper.NumberType.LONG); try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { { - InternalNested result = search(newSearcher(indexReader, false, true), + InternalNested result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType); InternalMultiBucketAggregation terms = result.getAggregations().get("terms"); @@ -1216,7 +1216,7 @@ public void testWithNestedAggregations() throws IOException { { FilterAggregationBuilder filter = new FilterAggregationBuilder("filter", new MatchAllQueryBuilder()) .subAggregation(nested); - InternalFilter result = search(newSearcher(indexReader, false, true), + InternalFilter result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), filter, fieldType); InternalNested nestedResult = result.getAggregations().get("nested"); @@ -1276,7 +1276,7 @@ private void threeLayerStringTestCase(String executionHint) throws IOException { TermsAggregationBuilder request = new TermsAggregationBuilder("i").field("i").executionHint(executionHint) .subAggregation(new TermsAggregationBuilder("j").field("j").executionHint(executionHint) .subAggregation(new TermsAggregationBuilder("k").field("k").executionHint(executionHint))); - StringTerms result = search(searcher, new MatchAllDocsQuery(), request, + StringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("i"), keywordField("j"), keywordField("k")); for (int i = 0; i < 10; i++) { StringTerms.Bucket iBucket = result.getBucketByKey(Integer.toString(i)); @@ -1316,7 +1316,7 @@ public void testThreeLayerLong() throws IOException { TermsAggregationBuilder request = new TermsAggregationBuilder("i").field("i") .subAggregation(new TermsAggregationBuilder("j").field("j") .subAggregation(new TermsAggregationBuilder("k").field("k"))); - LongTerms result = search(searcher, new MatchAllDocsQuery(), request, + LongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("i"), longField("j"), longField("k")); for (int i = 0; i < 10; i++) { LongTerms.Bucket iBucket = result.getBucketByKey(Integer.toString(i)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index cdcb17cbbfc92..b106835a85bea 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -266,7 +266,7 @@ public void testCase(MappedFieldType ft, ExtendedStatsAggregationBuilder aggBuilder = new ExtendedStatsAggregationBuilder("my_agg") .field("field") .sigma(randomDoubleBetween(0, 10, true)); - InternalExtendedStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalExtendedStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); verify.accept(stats); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index ca1cb92e0fd5c..f9a23d703097f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -57,7 +57,7 @@ public void testEmpty() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(Double.isInfinite(bounds.top)); assertTrue(Double.isInfinite(bounds.bottom)); assertTrue(Double.isInfinite(bounds.posLeft)); @@ -84,7 +84,7 @@ public void testUnmappedFieldWithDocs() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(Double.isInfinite(bounds.top)); assertTrue(Double.isInfinite(bounds.bottom)); assertTrue(Double.isInfinite(bounds.posLeft)); @@ -117,7 +117,7 @@ public void testMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, equalTo(lat)); assertThat(bounds.bottom, equalTo(lat)); assertThat(bounds.posLeft, equalTo(lon >= 0 ? lon : Double.POSITIVE_INFINITY)); @@ -144,7 +144,7 @@ public void testInvalidMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, - () -> search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); + () -> searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); assertThat(exception.getMessage(), startsWith("unsupported symbol")); } } @@ -194,7 +194,7 @@ public void testRandom() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index ab4a1fdd4a982..847c85fc1708e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -51,7 +51,7 @@ public void testEmpty() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); } @@ -71,11 +71,11 @@ public void testUnmapped() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); } @@ -97,7 +97,7 @@ public void testUnmappedWithMissing() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals(result.centroid(), expectedCentroid); assertTrue(AggregationInspectionHelper.hasValue(result)); } @@ -161,7 +161,7 @@ private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) thro .field("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals("my_agg", result.getName()); GeoPoint centroid = result.centroid(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 1c16c8b4d24ff..d2cd0be2ed3d8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -66,7 +66,7 @@ public void testEmpty() throws IOException { = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); @@ -90,7 +90,7 @@ public void testSimple() throws IOException { = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index d03e64e1b120d..8d8b3da169249 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -138,8 +138,11 @@ public static void initMockScripts() { return state; }); SCRIPTS.put("reduceScript", params -> { - List states = (List) params.get("states"); - return states.stream().mapToInt(Integer::intValue).sum(); + List states = (List) params.get("states"); + return states.stream() + .filter(a -> a instanceof Number) + .map(a -> (Number) a) + .mapToInt(Number::intValue).sum(); }); SCRIPTS.put("initScriptScore", params -> { @@ -262,10 +265,11 @@ public void testNoDocs() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT_NOOP).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); - assertEquals(0, ((HashMap) scriptedMetric.aggregation()).size()); + assertEquals(0, scriptedMetric.aggregation()); } } } @@ -282,7 +286,7 @@ public void testScriptedMetricWithoutCombine() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).reduceScript(REDUCE_SCRIPT); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); + () -> searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); assertEquals(exception.getMessage(), "[combineScript] must not be null: [scriptedMetric]"); } } @@ -300,7 +304,7 @@ public void testScriptedMetricWithoutReduce() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); + () -> searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); assertEquals(exception.getMessage(), "[reduceScript] must not be null: [scriptedMetric]"); } } @@ -321,7 +325,8 @@ public void testScriptedMetricWithCombine() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT) .combineScript(COMBINE_SCRIPT).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); assertEquals(numDocs, scriptedMetric.aggregation()); @@ -344,11 +349,12 @@ public void testScriptedMetricWithCombineAccessesScores() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT_SCORE).mapScript(MAP_SCRIPT_SCORE) .combineScript(COMBINE_SCRIPT_SCORE).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); // all documents have score of 1.0 - assertEquals((double) numDocs, scriptedMetric.aggregation()); + assertEquals(numDocs, scriptedMetric.aggregation()); } } } @@ -359,13 +365,16 @@ public void testScriptParamsPassedThrough() throws IOException { for (int i = 0; i < 100; i++) { indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); } + // force a single aggregator + indexWriter.forceMerge(1); } try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT_PARAMS).mapScript(MAP_SCRIPT_PARAMS) .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); // The result value depends on the script params. assertEquals(4896, scriptedMetric.aggregation()); @@ -414,7 +423,7 @@ public void testConflictingAggAndScriptParams() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Parameter name \"" + CONFLICTING_PARAM_NAME + "\" used in both aggregation and script parameters", ex.getMessage()); @@ -433,7 +442,7 @@ public void testSelfReferencingAggStateAfterInit() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs init script)", ex.getMessage()); } @@ -454,7 +463,7 @@ public void testSelfReferencingAggStateAfterMap() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs map script)", ex.getMessage()); } @@ -472,7 +481,7 @@ public void testSelfReferencingAggStateAfterCombine() throws IOException { .combineScript(COMBINE_SCRIPT_SELF_REF).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs combine script)", ex.getMessage()); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index 5e11dbf867bad..bb3902ad14d1e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -234,7 +234,7 @@ public void testPartiallyUnmapped() throws IOException { MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) { final IndexSearcher searcher = new IndexSearcher(multiReader); - final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, ft); + final InternalStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, ft); assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); @@ -433,7 +433,7 @@ private void testCase(StatsAggregationBuilder builder, try (IndexReader reader = indexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray); + final InternalStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray); verify.accept(stats); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java index ca1aebb6be135..1bf2a74c1b0e5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java @@ -256,7 +256,7 @@ public void testPartiallyUnmapped() throws IOException { final IndexSearcher searcher = newSearcher(multiReader, true, true); - final InternalSum internalSum = search(searcher, new MatchAllDocsQuery(), builder, fieldType); + final InternalSum internalSum = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(sum, internalSum.getValue(), 0d); assertTrue(AggregationInspectionHelper.hasValue(internalSum)); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 52c2bf92373ab..3d2acaff2840a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -65,7 +65,7 @@ public void testEmpty() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); @@ -88,7 +88,7 @@ public void testSimple() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java index 632ba6f6f48d9..eb7d87c3b0241 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java @@ -354,9 +354,6 @@ public void testDocCountDerivativeWithGaps() throws IOException { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -408,10 +405,6 @@ public void testDocCountDerivativeWithGaps_random() throws IOException { if (randomBoolean()) valueCounts_empty_rnd[i] = 0L; for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) { - - if (frequently()) { - indexWriter.commit(); - } document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); document.clear(); @@ -458,9 +451,6 @@ public void testDocCountDerivativeWithGaps_insertZeros() throws IOException { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -514,9 +504,6 @@ public void testSingleValueAggDerivativeWithGaps() throws Exception { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -634,10 +621,6 @@ public void testSingleValueAggDerivativeWithGaps_random() throws IOException { if (randomBoolean()) valueCounts_empty_rnd[i] = 0L; for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) { - - if (frequently()) { - indexWriter.commit(); - } document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); document.clear(); @@ -734,9 +717,6 @@ private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consume executeTestCase(query, aggBuilder, verify, indexWriter -> { Document document = new Document(); for (int i = 0; i < numValueBuckets; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i * interval)); indexWriter.addDocument(document); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java index 9d07a3deffe0c..2f9f37b1d9999 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java @@ -131,10 +131,6 @@ private void executeTestCase(Query query, Document document = new Document(); int counter = 0; for (String date : datasetTimes) { - if (frequently()) { - indexWriter.commit(); - } - long instant = asLong(date); document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); document.add(new LongPoint(INSTANT_FIELD, instant)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 691b515223c90..d30e60e7a3597 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -140,6 +140,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doAnswer; @@ -394,46 +395,6 @@ protected ScriptService getMockScriptService() { return null; } - protected A search(IndexSearcher searcher, - Query query, - AggregationBuilder builder, - MappedFieldType... fieldTypes) throws IOException { - return search(createIndexSettings(), searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes); - } - - protected A search(IndexSettings indexSettings, - IndexSearcher searcher, - Query query, - AggregationBuilder builder, - MappedFieldType... fieldTypes) throws IOException { - return search(indexSettings, searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes); - } - - protected A search(IndexSearcher searcher, - Query query, - AggregationBuilder builder, - int maxBucket, - MappedFieldType... fieldTypes) throws IOException { - return search(createIndexSettings(), searcher, query, builder, maxBucket, fieldTypes); - } - - protected A search(IndexSettings indexSettings, - IndexSearcher searcher, - Query query, - AggregationBuilder builder, - int maxBucket, - MappedFieldType... fieldTypes) throws IOException { - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C a = createAggregator(query, builder, searcher, indexSettings, bucketConsumer, fieldTypes); - a.preCollection(); - searcher.search(query, a); - a.postCollection(); - @SuppressWarnings("unchecked") - A result = (A) a.buildTopLevel(); - return result; - } - protected A searchAndReduce(IndexSearcher searcher, Query query, AggregationBuilder builder, @@ -469,74 +430,70 @@ protected A searchAndReduc int maxBucket, MappedFieldType... fieldTypes) throws IOException { final IndexReaderContext ctx = searcher.getTopReaderContext(); - - final ShardSearcher[] subSearchers; - if (ctx instanceof LeafReaderContext) { - subSearchers = new ShardSearcher[1]; - subSearchers[0] = new ShardSearcher((LeafReaderContext) ctx, ctx); - } else { - final CompositeReaderContext compCTX = (CompositeReaderContext) ctx; - final int size = compCTX.leaves().size(); - subSearchers = new ShardSearcher[size]; - for(int searcherIDX=0;searcherIDX aggs = new ArrayList<>(); Query rewritten = searcher.rewrite(query); - Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); - for (ShardSearcher subSearcher : subSearchers) { - MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C a = createAggregator(query, builder, subSearcher, indexSettings, shardBucketConsumer, fieldTypes); - a.preCollection(); - subSearcher.search(weight, a); - a.postCollection(); - InternalAggregation agg = a.buildTopLevel(); - aggs.add(agg); - } - if (aggs.isEmpty()) { - return (A) root.buildEmptyAggregation(); - } else { - if (randomBoolean() && aggs.size() > 1) { - // sometimes do an incremental reduce - int toReduceSize = aggs.size(); - Collections.shuffle(aggs, random()); - int r = randomIntBetween(1, toReduceSize); - List toReduce = aggs.subList(0, r); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction( - root.context().bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); - A reduced = (A) aggs.get(0).reduce(toReduce, context); - aggs = new ArrayList<>(aggs.subList(r, toReduceSize)); - aggs.add(reduced); + if (randomBoolean() && searcher.getIndexReader().leaves().size() > 0) { + assertThat(ctx, instanceOf(CompositeReaderContext.class)); + final CompositeReaderContext compCTX = (CompositeReaderContext) ctx; + final int size = compCTX.leaves().size(); + final ShardSearcher[] subSearchers = new ShardSearcher[size]; + for (int searcherIDX = 0; searcherIDX < subSearchers.length; searcherIDX++) { + final LeafReaderContext leave = compCTX.leaves().get(searcherIDX); + subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX); } - // now do the final reduce - MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( - root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); + for (ShardSearcher subSearcher : subSearchers) { + MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + C a = createAggregator(query, builder, subSearcher, indexSettings, shardBucketConsumer, fieldTypes); + a.preCollection(); + Weight weight = subSearcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); + subSearcher.search(weight, a); + a.postCollection(); + aggs.add(a.buildTopLevel()); + } + } else { + root.preCollection(); + searcher.search(rewritten, root); + root.postCollection(); + aggs.add(root.buildTopLevel()); + } + + if (randomBoolean() && aggs.size() > 1) { + // sometimes do an incremental reduce + int toReduceSize = aggs.size(); + Collections.shuffle(aggs, random()); + int r = randomIntBetween(1, toReduceSize); + List toReduce = aggs.subList(0, r); + InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction( + root.context().bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); + A reduced = (A) aggs.get(0).reduce(toReduce, context); + aggs = new ArrayList<>(aggs.subList(r, toReduceSize)); + aggs.add(reduced); + } + + // now do the final reduce + MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( + root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); - @SuppressWarnings("unchecked") - A internalAgg = (A) aggs.get(0).reduce(aggs, context); + @SuppressWarnings("unchecked") + A internalAgg = (A) aggs.get(0).reduce(aggs, context); - // materialize any parent pipelines - internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines); + // materialize any parent pipelines + internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines); - // materialize any sibling pipelines at top level - for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) { - internalAgg = (A) pipelineAggregator.reduce(internalAgg, context); - } - doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); - return internalAgg; + // materialize any sibling pipelines at top level + for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) { + internalAgg = (A) pipelineAggregator.reduce(internalAgg, context); } - + doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + return internalAgg; } protected void doAssertReducedMultiBucketConsumer(Aggregation agg, MultiBucketConsumerService.MultiBucketConsumer bucketConsumer) { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java index bddea56afb376..1675708122ef1 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java @@ -94,7 +94,7 @@ public void testSimple() throws IOException { MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("field", true, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index f594422d571a6..d46b75ad31f1d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -531,7 +531,7 @@ private InternalAggregation collect(AggregationBuilder builder, Query query, try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - return search(indexSearcher, query, builder, fields); + return searchAndReduce(indexSearcher, query, builder, fields); } } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 35eeece14f0cb..b603e1b40280e 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -654,7 +654,7 @@ protected void doNextSearch(SearchRequest request, ActionListener= 0 ? lon : Double.POSITIVE_INFINITY)); @@ -149,7 +149,7 @@ public void testInvalidMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); + () -> searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); assertThat(exception.getMessage(), startsWith("Unknown geometry type")); } } @@ -204,7 +204,7 @@ public void testRandomShapes() throws Exception { = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", true, true, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE));