From 5de0ed9432f4b41b697c9f3bc28953834cd3eae1 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 6 Aug 2020 14:08:26 +0200 Subject: [PATCH] Replace AggregatorTestCase#search with AggregatorTestCase#searchAndReduce (#60683) * Replace AggregatorTestCase#search with AggregatorTestCase#searchAndReduce This commit removes the ability to test the top level result of an aggregator before it runs the final reduce. All aggregator tests that use AggregatorTestCase#search are rewritten with AggregatorTestCase#searchAndReduce in order to ensure that we test the final output (the one sent to the end user) rather than an intermediary result that could be different. This change also removes spurious commits triggered on top of a random index writer. These commits slow down the tests and are redundant with the commits that the random index writer performs. --- .../stats/MatrixStatsAggregatorTests.java | 47 +----- .../ChildrenToParentAggregatorTests.java | 6 +- .../ParentToChildrenAggregatorTests.java | 5 +- .../InternalVariableWidthHistogram.java | 4 +- .../composite/CompositeAggregatorTests.java | 23 +-- .../bucket/filter/FilterAggregatorTests.java | 24 +-- .../bucket/filter/FiltersAggregatorTests.java | 58 +++---- .../AutoDateHistogramAggregatorTests.java | 116 ++++---------- .../DateHistogramAggregatorTests.java | 133 +++++---------- .../NumericHistogramAggregatorTests.java | 122 ++++++++------ .../RangeHistogramAggregatorTests.java | 90 +++++++---- ...VariableWidthHistogramAggregatorTests.java | 105 ++++-------- .../missing/MissingAggregatorTests.java | 18 +-- .../bucket/nested/NestedAggregatorTests.java | 23 +-- .../nested/ReverseNestedAggregatorTests.java | 8 +- .../bucket/range/IpRangeAggregatorTests.java | 6 +- .../sampler/DiversifiedSamplerTests.java | 4 +- .../terms/BinaryTermsAggregatorTests.java | 41 +---- .../terms/KeywordTermsAggregatorTests.java | 39 +---- .../terms/NumericTermsAggregatorTests.java | 43 +---- .../terms/RareTermsAggregatorTests.java | 120 +++++--------- .../SignificantTermsAggregatorTests.java | 29 ++-- .../bucket/terms/TermsAggregatorTests.java | 8 +- .../metrics/ExtendedStatsAggregatorTests.java | 2 +- .../metrics/GeoBoundsAggregatorTests.java | 10 +- .../metrics/GeoCentroidAggregatorTests.java | 10 +- .../HDRPercentileRanksAggregatorTests.java | 4 +- .../ScriptedMetricAggregatorTests.java | 37 +++-- .../metrics/StatsAggregatorTests.java | 4 +- .../metrics/SumAggregatorTests.java | 2 +- ...TDigestPercentileRanksAggregatorTests.java | 4 +- .../pipeline/DerivativeAggregatorTests.java | 20 --- .../pipeline/MovFnAggrgatorTests.java | 4 - .../aggregations/AggregatorTestCase.java | 151 +++++++----------- ...regatedPercentileRanksAggregatorTests.java | 2 +- .../topmetrics/TopMetricsAggregatorTests.java | 2 +- .../job/RollupIndexerIndexingTests.java | 2 +- .../GeoShapeCentroidAggregatorTests.java | 12 +- .../GeoShapeBoundsAggregatorTests.java | 10 +- 39 files changed, 480 insertions(+), 868 deletions(-) diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java index c6f33aa15db1d..4223931c4e32c 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java @@ -53,9 +53,9 @@ public void testNoData() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Collections.singletonList("field")); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); assertNull(stats.getStats()); - assertFalse(MatrixAggregationInspectionHelper.hasValue(stats)); + assertEquals(0L, stats.getDocCount()); } } } @@ -72,9 +72,9 @@ public void testUnmapped() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Collections.singletonList("bogus")); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); assertNull(stats.getStats()); - assertFalse(MatrixAggregationInspectionHelper.hasValue(stats)); + assertEquals(0L, stats.getDocCount()); } } } @@ -85,43 +85,6 @@ public void testTwoFields() throws Exception { String fieldB = "b"; MappedFieldType ftB = new NumberFieldMapper.NumberFieldType(fieldB, NumberFieldMapper.NumberType.DOUBLE); - try (Directory directory = newDirectory(); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { - - int numDocs = scaledRandomIntBetween(8192, 16384); - Double[] fieldAValues = new Double[numDocs]; - Double[] fieldBValues = new Double[numDocs]; - for (int docId = 0; docId < numDocs; docId++) { - Document document = new Document(); - fieldAValues[docId] = randomDouble(); - document.add(new SortedNumericDocValuesField(fieldA, NumericUtils.doubleToSortableLong(fieldAValues[docId]))); - - fieldBValues[docId] = randomDouble(); - document.add(new SortedNumericDocValuesField(fieldB, NumericUtils.doubleToSortableLong(fieldBValues[docId]))); - indexWriter.addDocument(document); - } - - MultiPassStats multiPassStats = new MultiPassStats(fieldA, fieldB); - multiPassStats.computeStats(Arrays.asList(fieldAValues), Arrays.asList(fieldBValues)); - try (IndexReader reader = indexWriter.getReader()) { - IndexSearcher searcher = new IndexSearcher(reader); - MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") - .fields(Arrays.asList(fieldA, fieldB)); - InternalMatrixStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ftA, ftB); - // Since `search` doesn't do any reduction, and the InternalMatrixStats object will have a null `MatrixStatsResults` - // object. That is created during the final reduction, which also does a final round of computations - // So we have to create a MatrixStatsResults object here manually so that the final `compute()` is called - multiPassStats.assertNearlyEqual(new MatrixStatsResults(stats.getStats())); - } - } - } - - public void testTwoFieldsReduce() throws Exception { - String fieldA = "a"; - MappedFieldType ftA = new NumberFieldMapper.NumberFieldType(fieldA, NumberFieldMapper.NumberType.DOUBLE); - String fieldB = "b"; - MappedFieldType ftB = new NumberFieldMapper.NumberFieldType(fieldB, NumberFieldMapper.NumberType.DOUBLE); - try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { @@ -145,8 +108,6 @@ public void testTwoFieldsReduce() throws Exception { MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg") .fields(Arrays.asList(fieldA, fieldB)); InternalMatrixStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ftA, ftB); - // Unlike testTwoFields, `searchAndReduce` will execute reductions so the `MatrixStatsResults` object - // will be populated and fully computed. We should use that value directly to test against multiPassStats.assertNearlyEqual(stats); assertTrue(MatrixAggregationInspectionHelper.hasValue(stats)); } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index 3a4a263a84b76..af4a0e0bc9530 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -303,7 +303,7 @@ private void testCase(Query query, IndexSearcher indexSearcher, Consumer buckets, List plan, } toMerge.add(buckets.get(startIdx)); // Don't remove the startIdx bucket because it will be replaced by the merged bucket - reduceContext.consumeBucketsAndMaybeBreak(- (toMerge.size() - 1)); + int toRemove = toMerge.stream().mapToInt(b -> countInnerBucket(b)+1).sum(); + reduceContext.consumeBucketsAndMaybeBreak(-toRemove + 1); Bucket merged_bucket = reduceBucket(toMerge, reduceContext); buckets.set(startIdx, merged_bucket); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index a2e6a16dbb7f8..4f81a955b9e28 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1879,7 +1879,7 @@ public void testEarlyTermination() throws Exception { ) ); - executeTestCase(true, false, new TermQuery(new Term("foo", "bar")), + executeTestCase(true, new TermQuery(new Term("foo", "bar")), dataset, () -> new CompositeAggregationBuilder("name", @@ -1899,7 +1899,7 @@ public void testEarlyTermination() throws Exception { ); // source field and index sorting config have different order - executeTestCase(true, false, new TermQuery(new Term("foo", "bar")), + executeTestCase(true, new TermQuery(new Term("foo", "bar")), dataset, () -> new CompositeAggregationBuilder("name", @@ -1936,7 +1936,7 @@ public void testIndexSortWithDuplicate() throws Exception { ); for (SortOrder order : SortOrder.values()) { - executeTestCase(true, false, new MatchAllDocsQuery(), + executeTestCase(true, new MatchAllDocsQuery(), dataset, () -> new CompositeAggregationBuilder("name", @@ -1959,7 +1959,7 @@ public void testIndexSortWithDuplicate() throws Exception { } ); - executeTestCase(true, false, new MatchAllDocsQuery(), + executeTestCase(true, new MatchAllDocsQuery(), dataset, () -> new CompositeAggregationBuilder("name", @@ -1989,14 +1989,12 @@ private void testSearchCase(List queries, Supplier create, Consumer verify) throws IOException { for (Query query : queries) { - executeTestCase(false, false, query, dataset, create, verify); - executeTestCase(false, true, query, dataset, create, verify); - executeTestCase(true, true, query, dataset, create, verify); + executeTestCase(false, query, dataset, create, verify); + executeTestCase(true, query, dataset, create, verify); } } private void executeTestCase(boolean useIndexSort, - boolean reduced, Query query, List>> dataset, Supplier create, @@ -2019,18 +2017,13 @@ private void executeTestCase(boolean useIndexSort, indexWriter.addDocument(document); document.clear(); } - if (reduced == false && randomBoolean()) { + if (rarely()) { indexWriter.forceMerge(1); } } try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); - final InternalComposite composite; - if (reduced) { - composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - } else { - composite = search(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - } + InternalComposite composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); verify.accept(composite); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java index 5afad0400149a..e1b1e951df470 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java @@ -62,7 +62,7 @@ public void testEmpty() throws Exception { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); QueryBuilder filter = QueryBuilders.termQuery("field", randomAlphaOfLength(5)); FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter); - InternalFilter response = search(indexSearcher, new MatchAllDocsQuery(), builder, + InternalFilter response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(response.getDocCount(), 0); assertFalse(AggregationInspectionHelper.hasValue(response)); @@ -80,7 +80,7 @@ public void testRandom() throws Exception { for (int i = 0; i < numDocs; i++) { if (frequently()) { // make sure we have more than one segment to test the merge - indexWriter.getReader().close(); + indexWriter.commit(); } int value = randomInt(maxTerm-1); expectedBucketCount[value] += 1; @@ -98,20 +98,12 @@ public void testRandom() throws Exception { QueryBuilder filter = QueryBuilders.termQuery("field", Integer.toString(value)); FilterAggregationBuilder builder = new FilterAggregationBuilder("test", filter); - for (boolean doReduce : new boolean[]{true, false}) { - final InternalFilter response; - if (doReduce) { - response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, - fieldType); - } else { - response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - assertEquals(response.getDocCount(), (long) expectedBucketCount[value]); - if (expectedBucketCount[value] > 0) { - assertTrue(AggregationInspectionHelper.hasValue(response)); - } else { - assertFalse(AggregationInspectionHelper.hasValue(response)); - } + final InternalFilter response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + assertEquals(response.getDocCount(), (long) expectedBucketCount[value]); + if (expectedBucketCount[value] > 0) { + assertTrue(AggregationInspectionHelper.hasValue(response)); + } else { + assertFalse(AggregationInspectionHelper.hasValue(response)); } } finally { indexReader.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 265115e720600..617eec9799a4d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -60,7 +60,7 @@ public void testEmpty() throws Exception { } FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters); builder.otherBucketKey("other"); - InternalFilters response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + InternalFilters response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(response.getBuckets().size(), numFilters); for (InternalFilters.InternalBucket filter : response.getBuckets()) { assertEquals(filter.getDocCount(), 0); @@ -113,22 +113,15 @@ public void testKeyedFilter() throws Exception { FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", keys); builder.otherBucket(true); builder.otherBucketKey("other"); - for (boolean doReduce : new boolean[] {true, false}) { - final InternalFilters filters; - if (doReduce) { - filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } else { - filters = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - assertEquals(filters.getBuckets().size(), 7); - assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2); - assertEquals(filters.getBucketByKey("foo").getDocCount(), 2); - assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2); - assertEquals(filters.getBucketByKey("bar").getDocCount(), 1); - assertEquals(filters.getBucketByKey("same").getDocCount(), 1); - assertEquals(filters.getBucketByKey("other").getDocCount(), 2); - assertTrue(AggregationInspectionHelper.hasValue(filters)); - } + final InternalFilters filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + assertEquals(filters.getBuckets().size(), 7); + assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2); + assertEquals(filters.getBucketByKey("foo").getDocCount(), 2); + assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2); + assertEquals(filters.getBucketByKey("bar").getDocCount(), 1); + assertEquals(filters.getBucketByKey("same").getDocCount(), 1); + assertEquals(filters.getBucketByKey("other").getDocCount(), 2); + assertTrue(AggregationInspectionHelper.hasValue(filters)); indexReader.close(); directory.close(); @@ -175,28 +168,21 @@ public void testRandom() throws Exception { builder.otherBucket(true); builder.otherBucketKey("other"); - for (boolean doReduce : new boolean[]{true, false}) { - final InternalFilters response; - if (doReduce) { - response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } else { - response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); - } - List buckets = response.getBuckets(); - assertEquals(buckets.size(), filters.length + 1); + final InternalFilters response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType); + List buckets = response.getBuckets(); + assertEquals(buckets.size(), filters.length + 1); - for (InternalFilters.InternalBucket bucket : buckets) { - if ("other".equals(bucket.getKey())) { - assertEquals(bucket.getDocCount(), expectedOtherCount); - } else { - int index = Integer.parseInt(bucket.getKey()); - assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]); - } + for (InternalFilters.InternalBucket bucket : buckets) { + if ("other".equals(bucket.getKey())) { + assertEquals(bucket.getDocCount(), expectedOtherCount); + } else { + int index = Integer.parseInt(bucket.getKey()); + assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]); } - - // Always true because we include 'other' in the agg - assertTrue(AggregationInspectionHelper.hasValue(response)); } + + // Always true because we include 'other' in the agg + assertTrue(AggregationInspectionHelper.hasValue(response)); } finally { indexReader.close(); directory.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index 477df9e59163c..4cb34acab2449 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -98,7 +98,7 @@ public class AutoDateHistogramAggregatorTests extends DateHistogramAggregatorTes private static final Query DEFAULT_QUERY = new MatchAllDocsQuery(); public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATES_WITH_TIME, + testSearchCase(new MatchNoDocsQuery(), DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -115,20 +115,16 @@ public void testMatchAllDocs() throws IOException { expectedDocCount.put("2015-01-01T00:00:00.000Z", 3); expectedDocCount.put("2016-01-01T00:00:00.000Z", 1); expectedDocCount.put("2017-01-01T00:00:00.000Z", 1); - testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2011-01-01T00:00:00.000Z", 0); expectedDocCount.put("2014-01-01T00:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); } public void testSubAggregations() throws IOException { - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD) .subAggregation(AggregationBuilders.stats("stats").field(DATE_FIELD)), histogram -> { @@ -249,7 +245,7 @@ public void testAsSubAgg() throws IOException { expectedMax.put("2020-01-01T00:00:00.000Z", 2.0); expectedMax.put("2021-01-01T00:00:00.000Z", 3.0); assertThat(maxAsMap(ak1adh), equalTo(expectedMax)); - + StringTerms.Bucket b = terms.getBucketByKey("b"); StringTerms bk1 = b.getAggregations().get("k1"); StringTerms.Bucket bk1a = bk1.getBucketByKey("a"); @@ -391,7 +387,7 @@ public void testNoDocs() throws IOException { assertFalse(AggregationInspectionHelper.hasValue(histogram)); } ); - testSearchAndReduceCase(DEFAULT_QUERY, dates, aggregation, + testSearchCase(DEFAULT_QUERY, dates, aggregation, histogram -> { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); @@ -431,7 +427,7 @@ public void testIntervalYear() throws IOException { final long start = LocalDate.of(2015, 1, 1).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); final long end = LocalDate.of(2017, 12, 31).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); final Query rangeQuery = LongPoint.newRangeQuery(INSTANT_FIELD, start, end); - testBothCases(rangeQuery, DATES_WITH_TIME, + testSearchCase(rangeQuery, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), histogram -> { final ZonedDateTime startDate = ZonedDateTime.of(2015, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); @@ -460,7 +456,7 @@ public void testIntervalMonth() throws IOException { expectedDocCount.put("2017-01-01T00:00:00.000Z", 1); expectedDocCount.put("2017-02-01T00:00:00.000Z", 2); expectedDocCount.put("2017-03-01T00:00:00.000Z", 3); - testBothCases(DEFAULT_QUERY, datesForMonthInterval, + testSearchCase(DEFAULT_QUERY, datesForMonthInterval, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -490,12 +486,8 @@ public void testIntervalDay() throws IOException { expectedDocCount.put("2017-02-02T00:00:00.000Z", 2); expectedDocCount.put("2017-02-03T00:00:00.000Z", 3); expectedDocCount.put("2017-02-05T00:00:00.000Z", 1); - testSearchCase(DEFAULT_QUERY, datesForDayInterval, - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-04T00:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + testSearchCase(DEFAULT_QUERY, datesForDayInterval, aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -515,12 +507,8 @@ public void testIntervalDayWithTZ() throws IOException { expectedDocCount.put("2017-02-01T00:00:00.000-01:00", 2); expectedDocCount.put("2017-02-02T00:00:00.000-01:00", 3); expectedDocCount.put("2017-02-04T00:00:00.000-01:00", 1); - testSearchCase(DEFAULT_QUERY, datesForDayInterval, - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-03T00:00:00.000-01:00", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + testSearchCase(DEFAULT_QUERY, datesForDayInterval, aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -546,13 +534,9 @@ public void testIntervalHour() throws IOException { expectedDocCount.put("2017-02-01T15:00:00.000Z", 1); expectedDocCount.put("2017-02-01T15:00:00.000Z", 1); expectedDocCount.put("2017-02-01T16:00:00.000Z", 3); - testSearchCase(DEFAULT_QUERY, datesForHourInterval, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T11:00:00.000Z", 0); expectedDocCount.put("2017-02-01T12:00:00.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -560,7 +544,7 @@ public void testIntervalHour() throws IOException { expectedDocCount.put("2017-02-01T09:00:00.000Z", 3); expectedDocCount.put("2017-02-01T12:00:00.000Z", 3); expectedDocCount.put("2017-02-01T15:00:00.000Z", 4); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -585,13 +569,9 @@ public void testIntervalHourWithTZ() throws IOException { expectedDocCount.put("2017-02-01T13:00:00.000-01:00", 2); expectedDocCount.put("2017-02-01T14:00:00.000-01:00", 1); expectedDocCount.put("2017-02-01T15:00:00.000-01:00", 3); - testSearchCase(DEFAULT_QUERY, datesForHourInterval, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T10:00:00.000-01:00", 0); expectedDocCount.put("2017-02-01T11:00:00.000-01:00", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + testSearchCase(DEFAULT_QUERY, datesForHourInterval, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD).timeZone(ZoneOffset.ofHours(-1)), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); @@ -612,7 +592,7 @@ public void testRandomSecondIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 30); bucketsToExpectedDocCountMap.put(3, 60); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -641,7 +621,7 @@ public void testRandomMinuteIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 30); bucketsToExpectedDocCountMap.put(3, 60); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -669,7 +649,7 @@ public void testRandomHourIntervals() throws IOException { bucketsToExpectedDocCountMap.put(12, 12); bucketsToExpectedDocCountMap.put(3, 24); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -693,7 +673,7 @@ public void testRandomDayIntervals() throws IOException { } final int randomChoice = randomIntBetween(1, 3); if (randomChoice == 1) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(length).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -704,7 +684,7 @@ public void testRandomDayIntervals() throws IOException { assertEquals(1, bucket.getDocCount()); }); } else if (randomChoice == 2) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(60).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -716,7 +696,7 @@ public void testRandomDayIntervals() throws IOException { assertEquals(expectedDocCount, bucket.getDocCount()); }); } else if (randomChoice == 3) { - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -742,7 +722,7 @@ public void testRandomMonthIntervals() throws IOException { bucketsToExpectedDocCountMap.put(30, 3); bucketsToExpectedDocCountMap.put(6, 12); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -772,7 +752,7 @@ public void testRandomYearIntervals() throws IOException { bucketsToExpectedDocCountMap.put(10, 50); bucketsToExpectedDocCountMap.put(5, 100); final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, + testSearchCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), histogram -> { final List buckets = histogram.getBuckets(); @@ -797,29 +777,20 @@ public void testIntervalMinute() throws IOException { skeletonDocCount.put("2017-02-01T09:02:00.000Z", 2); skeletonDocCount.put("2017-02-01T09:15:00.000Z", 1); skeletonDocCount.put("2017-02-01T09:16:00.000Z", 2); - testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(skeletonDocCount)) - ); Map fullDocCount = new TreeMap<>(); fullDocCount.put("2017-02-01T09:02:00.000Z", 2); fullDocCount.put("2017-02-01T09:07:00.000Z", 0); fullDocCount.put("2017-02-01T09:12:00.000Z", 3); - testSearchAndReduceCase(DEFAULT_QUERY, datesForMinuteInterval, + testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(fullDocCount)) ); - - testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, - aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(skeletonDocCount)) - ); fullDocCount.clear(); fullDocCount.putAll(skeletonDocCount); for (int minute = 3; minute < 15; minute++) { - fullDocCount.put(String.format(Locale.ROOT, "2017-02-01T09:%02d:00.000Z", minute), 0); + fullDocCount.put(String.format(Locale.ROOT, "2017-02-01T09:%02d:00.000Z", minute), 0); } - testSearchAndReduceCase(DEFAULT_QUERY, datesForMinuteInterval, + testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(fullDocCount)) ); @@ -837,22 +808,18 @@ public void testIntervalSecond() throws IOException { expectedDocCount.put("2017-02-01T00:00:05.000Z", 1); expectedDocCount.put("2017-02-01T00:00:07.000Z", 2); expectedDocCount.put("2017-02-01T00:00:11.000Z", 3); - testSearchCase(DEFAULT_QUERY, datesForSecondInterval, - aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), - result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) - ); expectedDocCount.put("2017-02-01T00:00:06.000Z", 0); expectedDocCount.put("2017-02-01T00:00:08.000Z", 0); expectedDocCount.put("2017-02-01T00:00:09.000Z", 0); expectedDocCount.put("2017-02-01T00:00:10.000Z", 0); - testSearchAndReduceCase(DEFAULT_QUERY, datesForSecondInterval, + testSearchCase(DEFAULT_QUERY, datesForSecondInterval, aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), result -> assertThat(bucketCountsAsMap(result), equalTo(expectedDocCount)) ); } public void testWithPipelineReductions() throws IOException { - testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, aggregation -> aggregation.setNumBuckets(1).field(DATE_FIELD) .subAggregation(AggregationBuilders.histogram("histo").field(NUMERIC_FIELD).interval(1) .subAggregation(AggregationBuilders.max("max").field(NUMERIC_FIELD)) @@ -880,25 +847,6 @@ public void testWithPipelineReductions() throws IOException { }); } - private void testSearchCase(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - } - - private void testSearchAndReduceCase(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, configure, verify); - } - - private void testBothCases(final Query query, final List dataset, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - executeTestCase(true, query, dataset, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { final Settings nodeSettings = Settings.builder() @@ -913,7 +861,7 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(final boolean reduced, final Query query, final List dataset, + private void testSearchCase(final Query query, final List dataset, final Consumer configure, final Consumer verify) throws IOException { try (Directory directory = newDirectory()) { @@ -936,12 +884,8 @@ private void executeTestCase(final boolean reduced, final Query query, final Lis MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NUMERIC_FIELD, NumberFieldMapper.NumberType.LONG); - final InternalAutoDateHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); - } + final InternalAutoDateHistogram histogram = + searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType, instantFieldType, numericFieldType); verify.accept(histogram); } } @@ -951,10 +895,6 @@ private void indexSampleData(List dataset, RandomIndexWriter inde final Document document = new Document(); int i = 0; for (final ZonedDateTime date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - final long instant = date.toInstant().toEpochMilli(); document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); document.add(new LongPoint(INSTANT_FIELD, instant)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 4d84a854fc57f..d297d9fabc613 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -67,7 +67,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas "2017-12-12T22:55:46"); public void testMatchNoDocsDeprecatedInterval() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -78,11 +78,11 @@ public void testMatchNoDocsDeprecatedInterval() throws IOException { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testBothCases(new MatchNoDocsQuery(), DATASET, + testSearchCase(new MatchNoDocsQuery(), DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -92,20 +92,13 @@ public void testMatchAllDocsDeprecatedInterval() throws IOException { Query query = new MatchAllDocsQuery(); testSearchCase(query, DATASET, - aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), - histogram -> { - assertEquals(6, histogram.getBuckets().size()); - assertTrue(AggregationInspectionHelper.hasValue(histogram)); - }, false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { assertEquals(8, histogram.getBuckets().size()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); }, false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> { assertEquals(6, histogram.getBuckets().size()); @@ -122,34 +115,26 @@ public void testMatchAllDocs() throws IOException { for (int i = 0; i < 1000; i++) { foo.add(DATASET.get(randomIntBetween(0, DATASET.size()-1))); } - testSearchAndReduceCase(query, foo, + testSearchCase(query, foo, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")) .field(AGGREGABLE_DATE).order(BucketOrder.count(false)), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); testSearchCase(query, DATASET, - aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), - histogram -> assertEquals(6, histogram.getBuckets().size()), false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); testSearchCase(query, DATASET, - aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), - histogram -> assertEquals(6, histogram.getBuckets().size()), false - ); - testSearchAndReduceCase(query, DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE), histogram -> assertEquals(8, histogram.getBuckets().size()), false ); - testBothCases(query, DATASET, + testSearchCase(query, DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field(AGGREGABLE_DATE).minDocCount(1L), histogram -> assertEquals(6, histogram.getBuckets().size()), false ); @@ -206,7 +191,7 @@ public void testNoDocsDeprecatedInterval() throws IOException { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); }, false); - testSearchAndReduceCase(query, dates, aggregation, histogram -> { + testSearchCase(query, dates, aggregation, histogram -> { assertEquals(0, histogram.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue(histogram)); }, false); @@ -221,7 +206,7 @@ public void testNoDocs() throws IOException { testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testSearchAndReduceCase(query, dates, aggregation, + testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); @@ -230,13 +215,13 @@ public void testNoDocs() throws IOException { testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testSearchAndReduceCase(query, dates, aggregation, + testSearchCase(query, dates, aggregation, histogram -> assertEquals(0, histogram.getBuckets().size()), false ); } public void testAggregateWrongFieldDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field("wrong_field"), histogram -> { assertEquals(0, histogram.getBuckets().size()); @@ -247,18 +232,18 @@ public void testAggregateWrongFieldDeprecated() throws IOException { } public void testAggregateWrongField() throws IOException { - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field("wrong_field"), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); - testBothCases(new MatchAllDocsQuery(), DATASET, + testSearchCase(new MatchAllDocsQuery(), DATASET, aggregation -> aggregation.fixedInterval(new DateHistogramInterval("365d")).field("wrong_field"), histogram -> assertEquals(0, histogram.getBuckets().size()), false ); } public void testIntervalYearDeprecated() throws IOException { - testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, + testSearchCase(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); @@ -281,7 +266,7 @@ public void testIntervalYearDeprecated() throws IOException { } public void testIntervalYear() throws IOException { - testBothCases(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, + testSearchCase(LongPoint.newRangeQuery(SEARCHABLE_DATE, asLong("2015-01-01"), asLong("2017-12-31")), DATASET, aggregation -> aggregation.calendarInterval(DateHistogramInterval.YEAR).field(AGGREGABLE_DATE), histogram -> { List buckets = histogram.getBuckets(); @@ -303,7 +288,7 @@ public void testIntervalYear() throws IOException { } public void testIntervalMonthDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { @@ -327,7 +312,7 @@ public void testIntervalMonthDeprecated() throws IOException { } public void testIntervalMonth() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), aggregation -> aggregation.calendarInterval(DateHistogramInterval.MONTH).field(AGGREGABLE_DATE), histogram -> { @@ -350,7 +335,7 @@ public void testIntervalMonth() throws IOException { } public void testIntervalDayDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -386,7 +371,7 @@ public void testIntervalDayDeprecated() throws IOException { } public void testIntervalDay() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -418,7 +403,7 @@ public void testIntervalDay() throws IOException { assertEquals(1, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01", "2017-02-02", @@ -453,7 +438,7 @@ public void testIntervalDay() throws IOException { } public void testIntervalHourDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -500,7 +485,7 @@ public void testIntervalHourDeprecated() throws IOException { } public void testIntervalHour() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -543,7 +528,7 @@ public void testIntervalHour() throws IOException { assertEquals(3, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:00.000Z", "2017-02-01T09:35:00.000Z", @@ -589,7 +574,7 @@ public void testIntervalHour() throws IOException { } public void testIntervalMinuteDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -619,7 +604,7 @@ public void testIntervalMinuteDeprecated() throws IOException { } public void testIntervalMinute() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -645,7 +630,7 @@ public void testIntervalMinute() throws IOException { assertEquals(2, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T09:02:35.000Z", "2017-02-01T09:02:59.000Z", @@ -674,7 +659,7 @@ public void testIntervalMinute() throws IOException { } public void testIntervalSecondDeprecated() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -705,7 +690,7 @@ public void testIntervalSecondDeprecated() throws IOException { } public void testIntervalSecond() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -732,7 +717,7 @@ public void testIntervalSecond() throws IOException { assertEquals(3, bucket.getDocCount()); }, false ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015Z", "2017-02-01T00:00:11.299Z", @@ -762,7 +747,7 @@ public void testIntervalSecond() throws IOException { } public void testNanosIntervalSecond() throws IOException { - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015298384Z", "2017-02-01T00:00:11.299954583Z", @@ -789,7 +774,7 @@ public void testNanosIntervalSecond() throws IOException { assertEquals(3, bucket.getDocCount()); }, true ); - testBothCases(new MatchAllDocsQuery(), + testSearchCase(new MatchAllDocsQuery(), Arrays.asList( "2017-02-01T00:00:05.015298384Z", "2017-02-01T00:00:11.299954583Z", @@ -829,7 +814,7 @@ public void testMinDocCountDeprecated() throws IOException { ); // 5 sec interval with minDocCount = 0 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); @@ -854,7 +839,7 @@ public void testMinDocCountDeprecated() throws IOException { ); // 5 sec interval with minDocCount = 3 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); @@ -879,7 +864,7 @@ public void testMinDocCount() throws IOException { ); // 5 sec interval with minDocCount = 0 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(0L), histogram -> { List buckets = histogram.getBuckets(); @@ -904,7 +889,7 @@ public void testMinDocCount() throws IOException { ); // 5 sec interval with minDocCount = 3 - testSearchAndReduceCase(query, timestamps, + testSearchCase(query, timestamps, aggregation -> aggregation.fixedInterval(DateHistogramInterval.seconds(5)).field(AGGREGABLE_DATE).minDocCount(3L), histogram -> { List buckets = histogram.getBuckets(); @@ -1175,43 +1160,6 @@ private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, int maxBucket, boolean useNanosecondResolution) throws IOException { - executeTestCase(false, query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, boolean useNanosecondResolution) throws IOException { - testSearchAndReduceCase(query, dataset, configure, verify, 1000, useNanosecondResolution); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - executeTestCase(true, query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, boolean useNanosecondResolution) throws IOException { - testBothCases(query, dataset, configure, verify, 10000, useNanosecondResolution); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - testSearchCase(query, dataset, configure, verify, maxBucket, useNanosecondResolution); - testSearchAndReduceCase(query, dataset, configure, verify, maxBucket, useNanosecondResolution); - } - - private void executeTestCase(boolean reduced, - Query query, - List dataset, - Consumer configure, - Consumer verify, - int maxBucket, boolean useNanosecondResolution) throws IOException { - boolean aggregableDateIsSearchable = randomBoolean(); DateFieldMapper.DateFieldType fieldType = aggregableDateFieldType(useNanosecondResolution, aggregableDateIsSearchable); @@ -1220,10 +1168,6 @@ private void executeTestCase(boolean reduced, try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (String date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long instant = asLong(date, fieldType); document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant)); if (aggregableDateIsSearchable) { @@ -1243,12 +1187,7 @@ private void executeTestCase(boolean reduced, configure.accept(aggregationBuilder); } - InternalDateHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, maxBucket, null, fieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, maxBucket, fieldType); - } + InternalDateHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, maxBucket, fieldType); verify.accept(histogram); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java index b41aa2896d3be..81426688c3e94 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregatorTests.java @@ -55,7 +55,7 @@ public class NumericHistogramAggregatorTests extends AggregatorTestCase { public void testLongs() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (long value : new long[] {7, 3, -10, -6, 5, 50}) { + for (long value : new long[] {7, 3, -10, -6, 5, 15}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); @@ -66,16 +66,20 @@ public void testLongs() throws Exception { .interval(5); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -84,7 +88,7 @@ public void testLongs() throws Exception { public void testDoubles() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (double value : new double[] {9.3, 3.2, -10, -6.5, 5.3, 50.1}) { + for (double value : new double[] {9.3, 3.2, -10, -6.5, 5.3, 15.1}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))); w.addDocument(doc); @@ -95,16 +99,21 @@ public void testDoubles() throws Exception { .interval(5); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(6, histogram.getBuckets().size()); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -130,10 +139,6 @@ public void testDates() throws Exception { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir)) { Document document = new Document(); for (String date : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long instant = fieldType.parse(date); document.add(new SortedNumericDocValuesField(fieldName, instant)); indexWriter.addDocument(document); @@ -145,7 +150,7 @@ public void testDates() throws Exception { .interval(1000 * 60 * 60 * 24); try (IndexReader reader = indexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -165,16 +170,20 @@ public void testIrrationalInterval() throws Exception { .interval(Math.PI); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-4 * Math.PI, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); assertEquals(-3 * Math.PI, histogram.getBuckets().get(1).getKey()); assertEquals(1, histogram.getBuckets().get(1).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(Math.PI, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-2 * Math.PI, histogram.getBuckets().get(2).getKey()); + assertEquals(0, histogram.getBuckets().get(2).getDocCount()); + assertEquals(-Math.PI, histogram.getBuckets().get(3).getKey()); + assertEquals(0, histogram.getBuckets().get(3).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(4).getKey()); + assertEquals(2, histogram.getBuckets().get(4).getDocCount()); + assertEquals(Math.PI, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -209,7 +218,7 @@ public void testMinDocCount() throws Exception { public void testMissing() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (long value : new long[] {7, 3, -10, -6, 5, 50}) { + for (long value : new long[] {7, 3, -10, -6, 5, 15}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); @@ -222,16 +231,20 @@ public void testMissing() throws Exception { .missing(2d); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); - assertEquals(4, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, longField("field")); + assertEquals(6, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(0d, histogram.getBuckets().get(1).getKey()); - assertEquals(7, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5d, histogram.getBuckets().get(2).getKey()); - assertEquals(2, histogram.getBuckets().get(2).getDocCount()); - assertEquals(50d, histogram.getBuckets().get(3).getKey()); - assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(7, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(1, histogram.getBuckets().get(5).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -252,7 +265,7 @@ public void testMissingUnmappedField() throws Exception { MappedFieldType type = null; try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, type); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, type); assertEquals(1, histogram.getBuckets().size()); @@ -280,7 +293,7 @@ public void testMissingUnmappedFieldBadType() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Throwable t = expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), aggBuilder, type); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, type); }); // This throws a number format exception (which is a subclass of IllegalArgumentException) and might be ok? assertThat(t.getMessage(), containsString(missingValue)); @@ -304,7 +317,7 @@ public void testIncorrectFieldType() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), aggBuilder, keywordField("field")); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, keywordField("field")); }); } } @@ -326,14 +339,16 @@ public void testOffset() throws Exception { .offset(Math.PI); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(3, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(4, histogram.getBuckets().size()); assertEquals(-10 + Math.PI, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); - assertEquals(Math.PI, histogram.getBuckets().get(1).getKey()); - assertEquals(2, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5 + Math.PI, histogram.getBuckets().get(2).getKey()); - assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(-5 + Math.PI, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(Math.PI, histogram.getBuckets().get(2).getKey()); + assertEquals(2, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5 + Math.PI, histogram.getBuckets().get(3).getKey()); + assertEquals(1, histogram.getBuckets().get(3).getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } @@ -358,18 +373,21 @@ public void testRandomOffset() throws Exception { .offset(offset); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); - assertEquals(3, histogram.getBuckets().size()); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, doubleField("field")); + assertEquals(4, histogram.getBuckets().size()); assertEquals(-10 + expectedOffset, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); - assertEquals(expectedOffset, histogram.getBuckets().get(1).getKey()); - assertEquals(1, histogram.getBuckets().get(1).getDocCount()); + assertEquals(-5 + expectedOffset, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); - assertEquals(5 + expectedOffset, histogram.getBuckets().get(2).getKey()); + assertEquals(expectedOffset, histogram.getBuckets().get(2).getKey()); assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5 + expectedOffset, histogram.getBuckets().get(3).getKey()); + assertEquals(1, histogram.getBuckets().get(3).getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java index 990c8840c3f9e..312ca43c7286e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregatorTests.java @@ -57,7 +57,7 @@ public void testDoubles() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -71,8 +71,9 @@ public void testDoubles() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -86,11 +87,14 @@ public void testDoubles() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -103,7 +107,7 @@ public void testLongs() throws Exception { new RangeFieldMapper.Range(rangeType, 1L, 5L, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3L, 4L, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4L, 13L, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42L, 49L, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22L, 29L, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -117,8 +121,9 @@ public void testLongs() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -132,11 +137,14 @@ public void testLongs() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -150,7 +158,7 @@ public void testMultipleRanges() throws Exception { new RangeFieldMapper.Range(rangeType, 1L, 5L, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3L, 4L, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4L, 13L, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42L, 49L, true, true) // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22L, 29L, true, true) // bucket 20, 25, 30 )); doc.add(new BinaryDocValuesField("field", encodedRange)); w.addDocument(doc); @@ -161,8 +169,9 @@ public void testMultipleRanges() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -176,11 +185,14 @@ public void testMultipleRanges() throws Exception { assertEquals(10d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d, histogram.getBuckets().get(5).getKey()); + assertEquals(20d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } @@ -206,7 +218,8 @@ public void testMultipleRangesLotsOfOverlap() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); assertEquals(3, histogram.getBuckets().size()); assertEquals(0d, histogram.getBuckets().get(0).getKey()); @@ -243,7 +256,8 @@ public void testLongsIrrationalInterval() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); assertEquals(6, histogram.getBuckets().size()); assertEquals(-1 * Math.PI, histogram.getBuckets().get(0).getKey()); @@ -315,7 +329,7 @@ public void testOffset() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket -1, 4 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -6 -1 4 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 4, 9 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 39, 44, 49 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 19, 24, 29 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -330,8 +344,9 @@ public void testOffset() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - //assertEquals(7, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(8, histogram.getBuckets().size()); assertEquals(-6d, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -345,14 +360,17 @@ public void testOffset() throws Exception { assertEquals(9d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(39d, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(14d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(44d, histogram.getBuckets().get(5).getKey()); + assertEquals(19d, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); - assertEquals(49d, histogram.getBuckets().get(6).getKey()); + assertEquals(24d, histogram.getBuckets().get(6).getKey()); assertEquals(1, histogram.getBuckets().get(6).getDocCount()); + + assertEquals(29d, histogram.getBuckets().get(7).getKey()); + assertEquals(1, histogram.getBuckets().get(7).getDocCount()); } } } @@ -365,7 +383,7 @@ public void testOffsetGtInterval() throws Exception { new RangeFieldMapper.Range(rangeType, 1.0D, 5.0D, true, true), // bucket 0 5 new RangeFieldMapper.Range(rangeType, -3.1, 4.2, true, true), // bucket -5, 0 new RangeFieldMapper.Range(rangeType, 4.2, 13.3, true, true), // bucket 0, 5, 10 - new RangeFieldMapper.Range(rangeType, 42.5, 49.3, true, true), // bucket 40, 45 + new RangeFieldMapper.Range(rangeType, 22.5, 29.3, true, true), // bucket 20, 25 }) { Document doc = new Document(); BytesRef encodedRange = rangeType.encodeRanges(Collections.singleton(range)); @@ -386,8 +404,9 @@ public void testOffsetGtInterval() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); - assertEquals(6, histogram.getBuckets().size()); + InternalHistogram histogram = + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType)); + assertEquals(7, histogram.getBuckets().size()); assertEquals(-5d + expectedOffset, histogram.getBuckets().get(0).getKey()); assertEquals(1, histogram.getBuckets().get(0).getDocCount()); @@ -401,11 +420,14 @@ public void testOffsetGtInterval() throws Exception { assertEquals(10d + expectedOffset, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); - assertEquals(40d + expectedOffset, histogram.getBuckets().get(4).getKey()); - assertEquals(1, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d + expectedOffset, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); - assertEquals(45d + expectedOffset, histogram.getBuckets().get(5).getKey()); + assertEquals(20d + expectedOffset, histogram.getBuckets().get(5).getKey()); assertEquals(1, histogram.getBuckets().get(5).getDocCount()); + + assertEquals(25d + expectedOffset, histogram.getBuckets().get(6).getKey()); + assertEquals(1, histogram.getBuckets().get(6).getDocCount()); } } } @@ -429,7 +451,7 @@ public void testIpRangesUnsupported() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Exception e = expectThrows(IllegalArgumentException.class, () -> - search(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType))); + searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, rangeField("field", rangeType))); assertThat(e.getMessage(), equalTo("Expected numeric range type but found non-numeric range [ip_range]")); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java index 8b24e92f6bb0e..c8fca52645a2c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java @@ -65,7 +65,7 @@ public class VariableWidthHistogramAggregatorTests extends AggregatorTestCase { public void testNoDocs() throws Exception{ final List dataset = Arrays.asList(); - testBothCases(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -87,7 +87,7 @@ public void testMoreClustersThanDocs() throws Exception { expectedMins.put(-3d, -3d); expectedMins.put(10d, 10d); - testBothCases(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -164,7 +164,7 @@ public void testDoubles() throws Exception { expectedMaxesOnlySearch.put(8.8, 8.8); testSearchCase(DEFAULT_QUERY, dataset, false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(6).setInitialBuffer(4), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedCentroidsOnlySearch.size(), buckets.size()); @@ -198,7 +198,7 @@ public void testDoubles() throws Exception { expectedMaxesSearchReduce.put(5.3, 5.9); expectedMaxesSearchReduce.put(8.8, 8.8); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, false, + testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(4).setShardSize(6).setInitialBuffer(4), histogram -> { final List buckets = histogram.getBuckets(); @@ -220,16 +220,12 @@ public void testNewBucketCreation() throws Exception { final List dataset = Arrays.asList(-1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 40, 30, 25, 32, 36, 80, 50, 75, 60); double doubleError = 1d / 10000d; - // Search (no reduce) - - // Expected clusters: [ (-1), (1), (3), (5), (7), (9), (11), (13), (15), (17), - // (19), (25, 30, 32), (36, 40, 50), (60), (75, 80) ] - // Corresponding keys (centroids): [ -1, 1, 3, ..., 17, 19, 29, 42, 77.5] - // Note: New buckets are created for 30, 50, and 80 because they are distant from the other buckets - final List keys = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 29d, 42d, 60d, 77.5d); - final List mins = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 25d, 36d, 60d, 75d); - final List maxes = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 32d, 50d, 60d, 80d); - final List docCounts = Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 2); + // Expected clusters: [ (-1, 1), (3, 5), (7, 9), (11, 13), (15, 17), + // (19), (25), (30), (32), (36), (40), (50), (60), (75), (80) ] + final List keys = Arrays.asList(0d, 4d, 8d, 12d, 16d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List mins = Arrays.asList(-1d, 3d, 7d, 11d, 15d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List maxes = Arrays.asList(1d, 5d, 9d, 13d, 17d, 19d, 25d, 30d, 32d, 36d, 40d, 50d, 60d, 75d, 80d); + final List docCounts = Arrays.asList(2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert keys.size() == docCounts.size() && keys.size() == keys.size(); final Map expectedDocCountOnlySearch = new HashMap<>(); @@ -242,7 +238,7 @@ public void testNewBucketCreation() throws Exception { } testSearchCase(DEFAULT_QUERY, dataset, false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(16).setInitialBuffer(12), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(15), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedDocCountOnlySearch.size(), buckets.size()); @@ -267,7 +263,7 @@ public void testNewBucketCreation() throws Exception { } testSearchCase(DEFAULT_QUERY, dataset.stream().map(n -> Double.valueOf(n.doubleValue() * Long.MAX_VALUE)).collect(toList()), false, - aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(16).setInitialBuffer(12), + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(15), histogram -> { final List buckets = histogram.getBuckets(); assertEquals(expectedDocCountOnlySearch.size(), buckets.size()); @@ -284,28 +280,22 @@ public void testNewBucketCreation() throws Exception { // There should not be more than `shard_size` documents on a node, even when very distant documents appear public void testNewBucketLimit() throws Exception{ - final List dataset = Arrays.asList(1,2,3,4,5, 10, 20, 50, 100, 5400, -900); + final List dataset = Arrays.asList(1, 2, 3, 4, 5, 10, 20, 50, 100, 5400, -900); double doubleError = 1d / 10000d; - // Expected clusters: [ (-900, 1, 2), (3, 4), (5), (10, 20, 50, 100, 5400)] - // Corresponding keys (centroids): [ -299, 3.5, 5, 1116] + // Expected clusters: [ (-900, 1, 2, 3, 4, 5), (10, 20, 50, 100, 5400)] + // Corresponding keys (centroids): [ -147.5, 1116] final Map expectedDocCount = new HashMap<>(); - expectedDocCount.put(-299d, 3); - expectedDocCount.put(3.5d, 2); - expectedDocCount.put(5d, 1); - expectedDocCount.put(1116d, 5); + expectedDocCount.put(-147.5d, 6); + expectedDocCount.put(1116.0d, 5); final Map expectedMins = new HashMap<>(); - expectedMins.put(-299d, -900d); - expectedMins.put(3.5d, 3d); - expectedMins.put(5d, 5d); - expectedMins.put(1116d, 10d); + expectedMins.put(-147.5d, -900d); + expectedMins.put(1116.0d, 10d); final Map expectedMaxes = new HashMap<>(); - expectedMaxes.put(-299d, 2d); - expectedMaxes.put(3.5d, 4d); - expectedMaxes.put(5d, 5d); - expectedMaxes.put(1116d, 5400d); + expectedMaxes.put(-147.5d, 5d); + expectedMaxes.put(1116.0d, 5400d); testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(2).setShardSize(4).setInitialBuffer(5), @@ -325,7 +315,7 @@ public void testNewBucketLimit() throws Exception{ public void testSimpleSubAggregations() throws IOException{ final List dataset = Arrays.asList(5, 1, 9, 2, 8); - testSearchAndReduceCase(DEFAULT_QUERY, dataset, false, + testSearchCase(DEFAULT_QUERY, dataset, false, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(3) .setInitialBuffer(3) @@ -426,7 +416,7 @@ public void testMultipleSegments() throws IOException{ // To account for this case of a document switching clusters, we check that each cluster centroid is within // a certain range, rather than asserting exact values. - testSearchAndReduceCase(DEFAULT_QUERY, dataset, true, + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD) .setNumBuckets(2) .setInitialBuffer(4) @@ -508,12 +498,11 @@ public void testSmallShardSize() throws Exception { public void testHugeShardSize() throws Exception { final List dataset = Arrays.asList(1, 2, 3); - testBothCases(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setShardSize(1000000000), histogram -> { - assertThat( + testSearchCase(DEFAULT_QUERY, dataset, true, aggregation -> aggregation.field(NUMERIC_FIELD).setShardSize(1000000000), + histogram -> assertThat( histogram.getBuckets().stream().map(InternalVariableWidthHistogram.Bucket::getKey).collect(toList()), - equalTo(List.of(1.0, 2.0, 3.0)) - ); - }); + equalTo(List.of(1.0, 2.0, 3.0))) + ); } public void testSmallInitialBuffer() throws Exception { @@ -529,7 +518,7 @@ public void testSmallInitialBuffer() throws Exception { public void testOutOfOrderInitialBuffer() throws Exception { final List dataset = Arrays.asList(1, 2, 3); - testBothCases( + testSearchCase( DEFAULT_QUERY, dataset, true, @@ -553,26 +542,6 @@ public void testDefaultInitialBufferDependsOnNumBuckets() throws Exception { assertThat(new VariableWidthHistogramAggregationBuilder("test").setNumBuckets(3).getInitialBuffer(), equalTo(1500)); } - private void testSearchCase(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(false, query, dataset, multipleSegments, configure, verify); - } - - - private void testSearchAndReduceCase(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, multipleSegments, configure, verify); - } - - private void testBothCases(final Query query, final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { - executeTestCase(true, query, dataset, multipleSegments, configure, verify); - executeTestCase(false, query, dataset, multipleSegments, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { final Settings nodeSettings = Settings.builder() @@ -588,10 +557,9 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(final boolean reduced, final Query query, - final List dataset, boolean multipleSegments, - final Consumer configure, - final Consumer verify) throws IOException { + private void testSearchCase(final Query query, final List dataset, boolean multipleSegments, + final Consumer configure, + final Consumer verify) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { indexSampleData(dataset, indexWriter, multipleSegments); @@ -619,12 +587,7 @@ private void executeTestCase(final boolean reduced, final Query query, - final InternalVariableWidthHistogram histogram; - if (reduced) { - histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); - } else { - histogram = search(indexSearcher, query, aggregationBuilder, fieldType); - } + final InternalVariableWidthHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); verify.accept(histogram); } } @@ -645,10 +608,6 @@ private void indexSampleData(List dataset, RandomIndexWriter indexWriter // Create multiple segments in the index final Document document = new Document(); for (final Number doc : dataset) { - if (frequently()) { - indexWriter.commit(); - } - long fieldVal = convertDocumentToSortableValue(doc); document.add(new SortedNumericDocValuesField(NUMERIC_FIELD, fieldVal)); indexWriter.addDocument(document); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 517f1bd07c8ae..d437e18788610 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -393,17 +393,6 @@ private void testCase(Query query, CheckedConsumer writeIndex, Consumer verify, Collection fieldTypes) throws IOException { - testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, false); - testCaseWithReduce(query, builder, writeIndex, verify, fieldTypes, true); - } - - private void testCaseWithReduce(Query query, - MissingAggregationBuilder builder, - CheckedConsumer writeIndex, - Consumer verify, - Collection fieldTypes, - boolean reduced) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { writeIndex.accept(indexWriter); @@ -412,12 +401,7 @@ private void testCaseWithReduce(Query query, try (IndexReader indexReader = DirectoryReader.open(directory)) { final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final InternalMissing missing; - if (reduced) { - missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); - } else { - missing = search(indexSearcher, query, builder, fieldTypesArray); - } + final InternalMissing missing = searchAndReduce(indexSearcher, query, builder, fieldTypesArray); verify.accept(missing); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index fde8b7e2e3518..078450445f546 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -151,7 +151,7 @@ public void testNoDocs() throws IOException { nestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(NESTED_AGG, nested.getName()); @@ -196,7 +196,7 @@ public void testSingleNestingMax() throws IOException { nestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -248,7 +248,7 @@ public void testDoubleNestingMax() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -303,7 +303,7 @@ public void testOrphanedDocs() throws IOException { nestedBuilder.subAggregation(sumAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -386,7 +386,7 @@ public void testResetRootDocId() throws Exception { bq.add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term(IdFieldMapper.NAME, Uid.encodeId("2"))), BooleanClause.Occur.MUST_NOT); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new ConstantScoreQuery(bq.build()), nestedBuilder, fieldType); assertEquals(NESTED_AGG, nested.getName()); @@ -424,7 +424,7 @@ public void testNestedOrdering() throws IOException { nestedBuilder.subAggregation(maxAgg); termsBuilder.subAggregation(nestedBuilder); - Terms terms = search(newSearcher(indexReader, false, true), + Terms terms = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(7, terms.getBuckets().size()); @@ -473,7 +473,8 @@ public void testNestedOrdering() throws IOException { nestedBuilder.subAggregation(maxAgg); termsBuilder.subAggregation(nestedBuilder); - terms = search(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); + terms = searchAndReduce(newSearcher(indexReader, false, true), + new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(7, terms.getBuckets().size()); assertEquals("authors", terms.getName()); @@ -561,7 +562,7 @@ public void testNestedOrdering_random() throws IOException { nestedBuilder.subAggregation(minAgg); termsBuilder.subAggregation(nestedBuilder); - Terms terms = search(newSearcher(indexReader, false, true), + Terms terms = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), termsBuilder, fieldType1, fieldType2); assertEquals(books.size(), terms.getBuckets().size()); @@ -658,7 +659,7 @@ public void testPreGetChildLeafCollectors() throws IOException { MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType("key"); MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType("value"); - Filter filter = search(newSearcher(indexReader, false, true), + Filter filter = searchAndReduce(newSearcher(indexReader, false, true), Queries.newNonNestedFilter(), filterAggregationBuilder, fieldType1, fieldType2); assertEquals("filterAgg", filter.getName()); @@ -719,9 +720,9 @@ public void testFieldAlias() throws IOException { NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation( max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias")); - InternalNested nested = search(newSearcher(indexReader, false, true), + InternalNested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), agg, fieldType); - Nested aliasNested = search(newSearcher(indexReader, false, true), + Nested aliasNested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), aliasAgg, fieldType); assertEquals(nested, aliasNested); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index 17e6c8e313f7f..dbcfa73d895b8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -98,7 +98,7 @@ public void testNoDocs() throws IOException { reverseNestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); ReverseNested reverseNested = (ReverseNested) ((InternalAggregation)nested).getProperty(REVERSE_AGG_NAME); @@ -160,7 +160,7 @@ public void testMaxFromParentDocs() throws IOException { reverseNestedBuilder.subAggregation(maxAgg); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), nestedBuilder, fieldType); assertEquals(expectedNestedDocs, nested.getDocCount()); @@ -224,9 +224,9 @@ public void testFieldAlias() throws IOException { NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation( reverseNested(REVERSE_AGG_NAME).subAggregation(aliasMaxAgg)); - Nested nested = search(newSearcher(indexReader, false, true), + Nested nested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), agg, fieldType); - Nested aliasNested = search(newSearcher(indexReader, false, true), + Nested aliasNested = searchAndReduce(newSearcher(indexReader, false, true), new MatchAllDocsQuery(), aliasAgg, fieldType); ReverseNested reverseNested = nested.getAggregations().get(REVERSE_AGG_NAME); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java index f4dd09c6eebd7..b811e150fe407 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java @@ -113,7 +113,7 @@ public void testRanges() throws Exception { MappedFieldType fieldType = new IpFieldMapper.IpFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, fieldType); + InternalBinaryRange range = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(numRanges, range.getBuckets().size()); for (int i = 0; i < range.getBuckets().size(); i++) { Tuple expected = requestedRanges[i]; @@ -148,7 +148,7 @@ public void testMissingUnmapped() throws Exception { .missing("192.168.100.42"); // Apparently we expect a string here try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); + InternalBinaryRange range = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); assertEquals(1, range.getBuckets().size()); } } @@ -169,7 +169,7 @@ public void testMissingUnmappedBadType() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); expectThrows(IllegalArgumentException.class, () -> { - search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); + searchAndReduce(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null); }); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java index f482342d25449..15ab2cc21a969 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerTests.java @@ -192,7 +192,7 @@ private void testCase(IndexSearcher indexSearcher, MappedFieldType genreFieldTyp .shardSize(shardSize) .subAggregation(new TermsAggregationBuilder("terms").field("id")); - InternalSampler result = search(indexSearcher, query, builder, genreFieldType, idFieldType); + InternalSampler result = searchAndReduce(indexSearcher, query, builder, genreFieldType, idFieldType); verify.accept(result); } @@ -211,7 +211,7 @@ public void testDiversifiedSampler_noDocs() throws Exception { .field(genreFieldType.name()) .subAggregation(new TermsAggregationBuilder("terms").field("id")); - InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType); + InternalSampler result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType); Terms terms = result.getAggregations().get("terms"); assertEquals(0, terms.getBuckets().size()); indexReader.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java index 8621475fa4c59..9f39e0e896d3a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java @@ -59,7 +59,7 @@ public class BinaryTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING ); @@ -68,7 +68,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -87,7 +87,7 @@ public void testBadIncludeExclude() throws IOException { // Make sure the include/exclude fails regardless of how the user tries to type hint the agg AggregationExecutionException e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), null // default, no hint )); @@ -95,7 +95,7 @@ public void testBadIncludeExclude() throws IOException { "they can only be applied to string fields. Use an array of values for include/exclude clauses")); e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), ValueType.STRING // string type hint )); @@ -104,7 +104,7 @@ public void testBadIncludeExclude() throws IOException { } public void testBadUserValueTypeHint() throws IOException { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testBothCases(new MatchNoDocsQuery(), dataset, + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(BINARY_FIELD), agg -> fail("test should have failed with exception"), ValueType.NUMERIC // numeric type hint )); @@ -114,34 +114,10 @@ public void testBadUserValueTypeHint() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (Long value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new BinaryFieldMapper.CustomBinaryDocValuesField(BINARY_FIELD, Numbers.longToBytes(value))); indexWriter.addDocument(document); document.clear(); @@ -161,12 +137,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType binaryFieldType = new BinaryFieldMapper.BinaryFieldType(BINARY_FIELD); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, binaryFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, binaryFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, binaryFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java index c3a1be37d05ba..4e049c7e3a176 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java @@ -56,12 +56,12 @@ public class KeywordTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING // with type hint ); @@ -70,7 +70,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -82,7 +82,7 @@ public void testMatchAllDocs() throws IOException { }, null // without type hint ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -98,34 +98,10 @@ public void testMatchAllDocs() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (String value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(value))); indexWriter.addDocument(document); document.clear(); @@ -145,12 +121,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType keywordFieldType = new KeywordFieldMapper.KeywordFieldType(KEYWORD_FIELD); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, keywordFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java index 30e32c7aa84bc..174a8a1aed6e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java @@ -59,12 +59,12 @@ public class NumericTermsAggregatorTests extends AggregatorTestCase { public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), null // without type hint ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> assertEquals(0, agg.getBuckets().size()), ValueType.NUMERIC // with type hint ); @@ -73,7 +73,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -85,7 +85,7 @@ public void testMatchAllDocs() throws IOException { }, null //without type hint ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD), agg -> { assertEquals(9, agg.getBuckets().size()); @@ -104,7 +104,7 @@ public void testBadIncludeExclude() throws IOException { // Numerics don't support any regex include/exclude, so should fail no matter what we do AggregationExecutionException e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), null )); @@ -113,7 +113,7 @@ public void testBadIncludeExclude() throws IOException { "values for include/exclude clauses used to filter numeric fields")); e = expectThrows(AggregationExecutionException.class, - () -> testBothCases(new MatchNoDocsQuery(), dataset, + () -> testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).includeExclude(includeExclude).format("yyyy-MM-dd"), agg -> fail("test should have failed with exception"), ValueType.NUMERIC // with type hint )); @@ -126,34 +126,10 @@ public void testBadIncludeExclude() throws IOException { private void testSearchCase(Query query, List dataset, Consumer configure, Consumer verify, ValueType valueType) throws IOException { - executeTestCase(false, query, dataset, configure, verify, valueType); - } - - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - executeTestCase(true, query, dataset, configure, verify, valueType); - } - - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - testSearchCase(query, dataset, configure, verify, valueType); - testSearchAndReduceCase(query, dataset, configure, verify, valueType); - } - - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify, ValueType valueType) throws IOException { - try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (Long value : dataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedNumericDocValuesField(LONG_FIELD, value)); document.add(new LongPoint(LONG_FIELD, value)); indexWriter.addDocument(document); @@ -175,12 +151,7 @@ private void executeTestCase(boolean reduced, Query query, List dataset, MappedFieldType longFieldType = new NumberFieldMapper.NumberFieldType(LONG_FIELD, NumberFieldMapper.NumberType.LONG); - InternalMappedTerms rareTerms; - if (reduced) { - rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, longFieldType); - } else { - rareTerms = search(indexSearcher, query, aggregationBuilder, longFieldType); - } + InternalMappedTerms rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, longFieldType); verify.accept(rareTerms); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index 17a61818e7768..78e8ca6f68147 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -99,11 +99,11 @@ public class RareTermsAggregatorTests extends AggregatorTestCase { } public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); - testBothCases(new MatchNoDocsQuery(), dataset, + testSearchCase(new MatchNoDocsQuery(), dataset, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); @@ -112,7 +112,7 @@ public void testMatchNoDocs() throws IOException { public void testMatchAllDocs() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -121,7 +121,7 @@ public void testMatchAllDocs() throws IOException { assertThat(bucket.getDocCount(), equalTo(1L)); } ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -144,7 +144,7 @@ public void testManyDocsOneRare() throws IOException { // The one rare term d.add(0L); - testSearchAndReduceCase(query, d, + testSearchCase(query, d, aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -153,7 +153,7 @@ public void testManyDocsOneRare() throws IOException { assertThat(bucket.getDocCount(), equalTo(1L)); } ); - testSearchAndReduceCase(query, d, + testSearchCase(query, d, aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> { assertEquals(1, agg.getBuckets().size()); @@ -167,7 +167,7 @@ public void testManyDocsOneRare() throws IOException { public void testIncludeExclude() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(LONG_FIELD) .maxDocCount(2) // bump to 2 since we're only including "2" .includeExclude(new IncludeExclude(new long[]{2}, new long[]{})), @@ -178,7 +178,7 @@ public void testIncludeExclude() throws IOException { assertThat(bucket.getDocCount(), equalTo(2L)); } ); - testBothCases(query, dataset, + testSearchCase(query, dataset, aggregation -> aggregation.field(KEYWORD_FIELD) .maxDocCount(2) // bump to 2 since we're only including "2" .includeExclude(new IncludeExclude(new String[]{"2"}, new String[]{})), @@ -194,7 +194,7 @@ public void testIncludeExclude() throws IOException { public void testEmbeddedMaxAgg() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD); aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(max); }, @@ -210,7 +210,7 @@ public void testEmbeddedMaxAgg() throws IOException { assertThat(((Max)(children.asList().get(0))).getValue(), equalTo(1.0)); } ); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD); aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(max); }, @@ -240,11 +240,11 @@ public void testEmpty() throws IOException { agg -> assertEquals(0, agg.getBuckets().size()) ); - testSearchAndReduceCase(query, Collections.emptyList(), + testSearchCase(query, Collections.emptyList(), aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); - testSearchAndReduceCase(query, Collections.emptyList(), + testSearchCase(query, Collections.emptyList(), aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1), agg -> assertEquals(0, agg.getBuckets().size()) ); @@ -312,7 +312,7 @@ public void testRangeField() throws Exception { public void testNestedTerms() throws IOException { Query query = new MatchAllDocsQuery(); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms") .field(KEYWORD_FIELD); aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(terms); @@ -331,7 +331,7 @@ public void testNestedTerms() throws IOException { } ); - testBothCases(query, dataset, aggregation -> { + testSearchCase(query, dataset, aggregation -> { TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms") .field(KEYWORD_FIELD); aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(terms); @@ -352,22 +352,20 @@ public void testNestedTerms() throws IOException { } public void testInsideTerms() throws IOException { - for (boolean reduce : new boolean[] {false, true}) { - for (String field : new String[] {KEYWORD_FIELD, LONG_FIELD}) { - AggregationBuilder builder = new TermsAggregationBuilder("terms").field("even_odd").subAggregation( - new RareTermsAggregationBuilder("rare").field(field).maxDocCount(2)); - StringTerms terms = (StringTerms) executeTestCase(reduce, new MatchAllDocsQuery(), dataset, builder); - - StringTerms.Bucket even = terms.getBucketByKey("even"); - InternalRareTerms evenRare = even.getAggregations().get("rare"); - assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("2")); - assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(2L)); - - StringTerms.Bucket odd = terms.getBucketByKey("odd"); - InternalRareTerms oddRare = odd.getAggregations().get("rare"); - assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("1")); - assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(1L)); - } + for (String field : new String[] {KEYWORD_FIELD, LONG_FIELD}) { + AggregationBuilder builder = new TermsAggregationBuilder("terms").field("even_odd").subAggregation( + new RareTermsAggregationBuilder("rare").field(field).maxDocCount(2)); + StringTerms terms = executeTestCase(new MatchAllDocsQuery(), dataset, builder); + + StringTerms.Bucket even = terms.getBucketByKey("even"); + InternalRareTerms evenRare = even.getAggregations().get("rare"); + assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("2")); + assertEquals(evenRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(2L)); + + StringTerms.Bucket odd = terms.getBucketByKey("odd"); + InternalRareTerms oddRare = odd.getAggregations().get("rare"); + assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getKeyAsString).collect(toList()), List.of("1")); + assertEquals(oddRare.getBuckets().stream().map(InternalRareTerms.Bucket::getDocCount).collect(toList()), List.of(1L)); } } @@ -530,34 +528,6 @@ private List generateDocsWithNested(String id, int value, int[] nested return documents; } - private void testSearchCase( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - executeTestCase(false, query, dataset, configure, verify); - } - - private void testSearchAndReduceCase( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - executeTestCase(true, query, dataset, configure, verify); - } - - private void testBothCases( - Query query, - List dataset, - Consumer configure, - Consumer> verify - ) throws IOException { - testSearchCase(query, dataset, configure, verify); - testSearchAndReduceCase(query, dataset, configure, verify); - } - @Override protected IndexSettings createIndexSettings() { Settings nodeSettings = Settings.builder() @@ -572,38 +542,27 @@ protected IndexSettings createIndexSettings() { ); } - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer> verify) throws IOException { + private void testSearchCase(Query query, + List dataset, + Consumer configure, + Consumer> verify) throws IOException { RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name"); if (configure != null) { configure.accept(aggregationBuilder); } - InternalMappedRareTerms result = (InternalMappedRareTerms) executeTestCase( - reduced, - query, - dataset, - aggregationBuilder - ); - verify.accept(result); + verify.accept(executeTestCase(query, dataset, aggregationBuilder)); + } - private InternalAggregation executeTestCase( - boolean reduced, - Query query, - List dataset, - AggregationBuilder aggregationBuilder - ) throws IOException { + private A executeTestCase(Query query, + List dataset, + AggregationBuilder aggregationBuilder) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); List shuffledDataset = new ArrayList<>(dataset); Collections.shuffle(shuffledDataset, random()); for (Long value : shuffledDataset) { - if (frequently()) { - indexWriter.commit(); - } - document.add(new SortedNumericDocValuesField(LONG_FIELD, value)); document.add(new LongPoint(LONG_FIELD, value)); document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(Long.toString(value)))); @@ -620,10 +579,7 @@ private InternalAggregation executeTestCase( keywordField(KEYWORD_FIELD), longField(LONG_FIELD), keywordField("even_odd")}; - if (reduced) { - return searchAndReduce(indexSearcher, query, aggregationBuilder, types); - } - return search(indexSearcher, query, aggregationBuilder, types); + return searchAndReduce(indexSearcher, query, aggregationBuilder, types); } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index 59bdda66dea41..e992eab92bf04 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -115,7 +115,7 @@ public void testSignificance() throws IOException { indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("text"); sigAgg.executionHint(randomExecutionHint()); @@ -259,7 +259,7 @@ public void testUnmapped() throws IOException { indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); // Attempt aggregation on unmapped field SignificantTermsAggregationBuilder sigAgg = new SignificantTermsAggregationBuilder("sig_text").field("unmapped_field"); @@ -328,7 +328,7 @@ public void testFieldAlias() throws IOException { indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { - addMixedTextDocs(textFieldType, w); + addMixedTextDocs(w); SignificantTermsAggregationBuilder agg = significantTerms("sig_text").field("text"); SignificantTermsAggregationBuilder aliasAgg = significantTerms("sig_text").field("text-alias"); @@ -389,7 +389,7 @@ private void testAllDocsWithoutStringField(String executionHint) throws IOExcept IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f") .executionHint(executionHint); - SignificantStringTerms result = search(searcher, new MatchAllDocsQuery(), request, keywordField("f")); + SignificantStringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("f")); assertThat(result.getSubsetSize(), equalTo(1L)); } } @@ -409,7 +409,7 @@ public void testAllDocsWithoutNumericField() throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f"); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, longField("f")); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("f")); assertThat(result.getSubsetSize(), equalTo(1L)); } } @@ -441,7 +441,7 @@ private void testSomeDocsWithoutStringField(String executionHint) throws IOExcep IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f") .executionHint(executionHint); - SignificantStringTerms result = search(searcher, new MatchAllDocsQuery(), request, keywordField("f")); + SignificantStringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("f")); assertThat(result.getSubsetSize(), equalTo(2L)); } } @@ -463,7 +463,7 @@ public void testSomeDocsWithoutNumericField() throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("f").field("f"); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, longField("f")); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("f")); assertThat(result.getSubsetSize(), equalTo(2L)); } } @@ -495,14 +495,17 @@ private void threeLayerStringTestCase(String executionHint) throws IOException { try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); SignificantTermsAggregationBuilder kRequest = new SignificantTermsAggregationBuilder("k").field("k") + .minDocCount(0) .executionHint(executionHint); SignificantTermsAggregationBuilder jRequest = new SignificantTermsAggregationBuilder("j").field("j") + .minDocCount(0) .executionHint(executionHint) .subAggregation(kRequest); SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i") + .minDocCount(0) .executionHint(executionHint) .subAggregation(jRequest); - SignificantStringTerms result = search( + SignificantStringTerms result = searchAndReduce( searcher, new MatchAllDocsQuery(), request, @@ -549,10 +552,10 @@ public void testThreeLayerLong() throws IOException { } try (IndexReader reader = maybeWrapReaderEs(writer.getReader())) { IndexSearcher searcher = newIndexSearcher(reader); - SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i") - .subAggregation(new SignificantTermsAggregationBuilder("j").field("j") - .subAggregation(new SignificantTermsAggregationBuilder("k").field("k"))); - SignificantLongTerms result = search(searcher, new MatchAllDocsQuery(), request, + SignificantTermsAggregationBuilder request = new SignificantTermsAggregationBuilder("i").field("i").minDocCount(0) + .subAggregation(new SignificantTermsAggregationBuilder("j").field("j").minDocCount(0) + .subAggregation(new SignificantTermsAggregationBuilder("k").field("k").minDocCount(0))); + SignificantLongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("i"), longField("j"), longField("k")); assertThat(result.getSubsetSize(), equalTo(1000L)); for (int i = 0; i < 10; i++) { @@ -576,7 +579,7 @@ public void testThreeLayerLong() throws IOException { } } - private void addMixedTextDocs(TextFieldType textFieldType, IndexWriter w) throws IOException { + private void addMixedTextDocs(IndexWriter w) throws IOException { for (int i = 0; i < 10; i++) { Document doc = new Document(); StringBuilder text = new StringBuilder("common "); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index ad2304c58d9f5..45a7db50cc0f5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -1206,7 +1206,7 @@ public void testWithNestedAggregations() throws IOException { = new NumberFieldMapper.NumberFieldType("nested_value", NumberFieldMapper.NumberType.LONG); try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { { - InternalNested result = search(newSearcher(indexReader, false, true), + InternalNested result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType); InternalMultiBucketAggregation terms = result.getAggregations().get("terms"); @@ -1216,7 +1216,7 @@ public void testWithNestedAggregations() throws IOException { { FilterAggregationBuilder filter = new FilterAggregationBuilder("filter", new MatchAllQueryBuilder()) .subAggregation(nested); - InternalFilter result = search(newSearcher(indexReader, false, true), + InternalFilter result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), filter, fieldType); InternalNested nestedResult = result.getAggregations().get("nested"); @@ -1276,7 +1276,7 @@ private void threeLayerStringTestCase(String executionHint) throws IOException { TermsAggregationBuilder request = new TermsAggregationBuilder("i").field("i").executionHint(executionHint) .subAggregation(new TermsAggregationBuilder("j").field("j").executionHint(executionHint) .subAggregation(new TermsAggregationBuilder("k").field("k").executionHint(executionHint))); - StringTerms result = search(searcher, new MatchAllDocsQuery(), request, + StringTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, keywordField("i"), keywordField("j"), keywordField("k")); for (int i = 0; i < 10; i++) { StringTerms.Bucket iBucket = result.getBucketByKey(Integer.toString(i)); @@ -1316,7 +1316,7 @@ public void testThreeLayerLong() throws IOException { TermsAggregationBuilder request = new TermsAggregationBuilder("i").field("i") .subAggregation(new TermsAggregationBuilder("j").field("j") .subAggregation(new TermsAggregationBuilder("k").field("k"))); - LongTerms result = search(searcher, new MatchAllDocsQuery(), request, + LongTerms result = searchAndReduce(searcher, new MatchAllDocsQuery(), request, longField("i"), longField("j"), longField("k")); for (int i = 0; i < 10; i++) { LongTerms.Bucket iBucket = result.getBucketByKey(Integer.toString(i)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index cdcb17cbbfc92..b106835a85bea 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -266,7 +266,7 @@ public void testCase(MappedFieldType ft, ExtendedStatsAggregationBuilder aggBuilder = new ExtendedStatsAggregationBuilder("my_agg") .field("field") .sigma(randomDoubleBetween(0, 10, true)); - InternalExtendedStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft); + InternalExtendedStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, ft); verify.accept(stats); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index ca1cb92e0fd5c..f9a23d703097f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -57,7 +57,7 @@ public void testEmpty() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(Double.isInfinite(bounds.top)); assertTrue(Double.isInfinite(bounds.bottom)); assertTrue(Double.isInfinite(bounds.posLeft)); @@ -84,7 +84,7 @@ public void testUnmappedFieldWithDocs() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertTrue(Double.isInfinite(bounds.top)); assertTrue(Double.isInfinite(bounds.bottom)); assertTrue(Double.isInfinite(bounds.posLeft)); @@ -117,7 +117,7 @@ public void testMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, equalTo(lat)); assertThat(bounds.bottom, equalTo(lat)); assertThat(bounds.posLeft, equalTo(lon >= 0 ? lon : Double.POSITIVE_INFINITY)); @@ -144,7 +144,7 @@ public void testInvalidMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, - () -> search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); + () -> searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); assertThat(exception.getMessage(), startsWith("unsupported symbol")); } } @@ -194,7 +194,7 @@ public void testRandom() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index ab4a1fdd4a982..847c85fc1708e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -51,7 +51,7 @@ public void testEmpty() throws Exception { MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); } @@ -71,11 +71,11 @@ public void testUnmapped() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); } @@ -97,7 +97,7 @@ public void testUnmappedWithMissing() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType("another_field"); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals(result.centroid(), expectedCentroid); assertTrue(AggregationInspectionHelper.hasValue(result)); } @@ -161,7 +161,7 @@ private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) thro .field("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals("my_agg", result.getName()); GeoPoint centroid = result.centroid(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 1c16c8b4d24ff..d2cd0be2ed3d8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -66,7 +66,7 @@ public void testEmpty() throws IOException { = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); @@ -90,7 +90,7 @@ public void testSimple() throws IOException { = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index d03e64e1b120d..8d8b3da169249 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -138,8 +138,11 @@ public static void initMockScripts() { return state; }); SCRIPTS.put("reduceScript", params -> { - List states = (List) params.get("states"); - return states.stream().mapToInt(Integer::intValue).sum(); + List states = (List) params.get("states"); + return states.stream() + .filter(a -> a instanceof Number) + .map(a -> (Number) a) + .mapToInt(Number::intValue).sum(); }); SCRIPTS.put("initScriptScore", params -> { @@ -262,10 +265,11 @@ public void testNoDocs() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT_NOOP).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); - assertEquals(0, ((HashMap) scriptedMetric.aggregation()).size()); + assertEquals(0, scriptedMetric.aggregation()); } } } @@ -282,7 +286,7 @@ public void testScriptedMetricWithoutCombine() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).reduceScript(REDUCE_SCRIPT); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); + () -> searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); assertEquals(exception.getMessage(), "[combineScript] must not be null: [scriptedMetric]"); } } @@ -300,7 +304,7 @@ public void testScriptedMetricWithoutReduce() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); + () -> searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder)); assertEquals(exception.getMessage(), "[reduceScript] must not be null: [scriptedMetric]"); } } @@ -321,7 +325,8 @@ public void testScriptedMetricWithCombine() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT) .combineScript(COMBINE_SCRIPT).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); assertEquals(numDocs, scriptedMetric.aggregation()); @@ -344,11 +349,12 @@ public void testScriptedMetricWithCombineAccessesScores() throws IOException { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT_SCORE).mapScript(MAP_SCRIPT_SCORE) .combineScript(COMBINE_SCRIPT_SCORE).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); // all documents have score of 1.0 - assertEquals((double) numDocs, scriptedMetric.aggregation()); + assertEquals(numDocs, scriptedMetric.aggregation()); } } } @@ -359,13 +365,16 @@ public void testScriptParamsPassedThrough() throws IOException { for (int i = 0; i < 100; i++) { indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); } + // force a single aggregator + indexWriter.forceMerge(1); } try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT_PARAMS).mapScript(MAP_SCRIPT_PARAMS) .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); - ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); + ScriptedMetric scriptedMetric = + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); // The result value depends on the script params. assertEquals(4896, scriptedMetric.aggregation()); @@ -414,7 +423,7 @@ public void testConflictingAggAndScriptParams() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Parameter name \"" + CONFLICTING_PARAM_NAME + "\" used in both aggregation and script parameters", ex.getMessage()); @@ -433,7 +442,7 @@ public void testSelfReferencingAggStateAfterInit() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs init script)", ex.getMessage()); } @@ -454,7 +463,7 @@ public void testSelfReferencingAggStateAfterMap() throws IOException { .combineScript(COMBINE_SCRIPT_PARAMS).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs map script)", ex.getMessage()); } @@ -472,7 +481,7 @@ public void testSelfReferencingAggStateAfterCombine() throws IOException { .combineScript(COMBINE_SCRIPT_SELF_REF).reduceScript(REDUCE_SCRIPT); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> - search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) + searchAndReduce(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder) ); assertEquals("Iterable object is self-referencing itself (Scripted metric aggs combine script)", ex.getMessage()); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index 5e11dbf867bad..bb3902ad14d1e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -234,7 +234,7 @@ public void testPartiallyUnmapped() throws IOException { MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) { final IndexSearcher searcher = new IndexSearcher(multiReader); - final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, ft); + final InternalStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, ft); assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); @@ -433,7 +433,7 @@ private void testCase(StatsAggregationBuilder builder, try (IndexReader reader = indexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]); - final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray); + final InternalStats stats = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray); verify.accept(stats); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java index ca1aebb6be135..1bf2a74c1b0e5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java @@ -256,7 +256,7 @@ public void testPartiallyUnmapped() throws IOException { final IndexSearcher searcher = newSearcher(multiReader, true, true); - final InternalSum internalSum = search(searcher, new MatchAllDocsQuery(), builder, fieldType); + final InternalSum internalSum = searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldType); assertEquals(sum, internalSum.getValue(), 0d); assertTrue(AggregationInspectionHelper.hasValue(internalSum)); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 52c2bf92373ab..3d2acaff2840a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -65,7 +65,7 @@ public void testEmpty() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); @@ -88,7 +88,7 @@ public void testSimple() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java index 632ba6f6f48d9..eb7d87c3b0241 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java @@ -354,9 +354,6 @@ public void testDocCountDerivativeWithGaps() throws IOException { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -408,10 +405,6 @@ public void testDocCountDerivativeWithGaps_random() throws IOException { if (randomBoolean()) valueCounts_empty_rnd[i] = 0L; for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) { - - if (frequently()) { - indexWriter.commit(); - } document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); document.clear(); @@ -458,9 +451,6 @@ public void testDocCountDerivativeWithGaps_insertZeros() throws IOException { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -514,9 +504,6 @@ public void testSingleValueAggDerivativeWithGaps() throws Exception { indexWriter -> { Document document = new Document(); for (int i = 0; i < valueCounts_empty.length; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts_empty[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); @@ -634,10 +621,6 @@ public void testSingleValueAggDerivativeWithGaps_random() throws IOException { if (randomBoolean()) valueCounts_empty_rnd[i] = 0L; for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) { - - if (frequently()) { - indexWriter.commit(); - } document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i)); indexWriter.addDocument(document); document.clear(); @@ -734,9 +717,6 @@ private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consume executeTestCase(query, aggBuilder, verify, indexWriter -> { Document document = new Document(); for (int i = 0; i < numValueBuckets; i++) { - if (frequently()) { - indexWriter.commit(); - } for (int docs = 0; docs < valueCounts[i]; docs++) { document.add(new NumericDocValuesField(SINGLE_VALUED_FIELD_NAME, i * interval)); indexWriter.addDocument(document); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java index 9d07a3deffe0c..2f9f37b1d9999 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java @@ -131,10 +131,6 @@ private void executeTestCase(Query query, Document document = new Document(); int counter = 0; for (String date : datasetTimes) { - if (frequently()) { - indexWriter.commit(); - } - long instant = asLong(date); document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); document.add(new LongPoint(INSTANT_FIELD, instant)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 691b515223c90..d30e60e7a3597 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -140,6 +140,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doAnswer; @@ -394,46 +395,6 @@ protected ScriptService getMockScriptService() { return null; } - protected A search(IndexSearcher searcher, - Query query, - AggregationBuilder builder, - MappedFieldType... fieldTypes) throws IOException { - return search(createIndexSettings(), searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes); - } - - protected A search(IndexSettings indexSettings, - IndexSearcher searcher, - Query query, - AggregationBuilder builder, - MappedFieldType... fieldTypes) throws IOException { - return search(indexSettings, searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes); - } - - protected A search(IndexSearcher searcher, - Query query, - AggregationBuilder builder, - int maxBucket, - MappedFieldType... fieldTypes) throws IOException { - return search(createIndexSettings(), searcher, query, builder, maxBucket, fieldTypes); - } - - protected A search(IndexSettings indexSettings, - IndexSearcher searcher, - Query query, - AggregationBuilder builder, - int maxBucket, - MappedFieldType... fieldTypes) throws IOException { - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C a = createAggregator(query, builder, searcher, indexSettings, bucketConsumer, fieldTypes); - a.preCollection(); - searcher.search(query, a); - a.postCollection(); - @SuppressWarnings("unchecked") - A result = (A) a.buildTopLevel(); - return result; - } - protected A searchAndReduce(IndexSearcher searcher, Query query, AggregationBuilder builder, @@ -469,74 +430,70 @@ protected A searchAndReduc int maxBucket, MappedFieldType... fieldTypes) throws IOException { final IndexReaderContext ctx = searcher.getTopReaderContext(); - - final ShardSearcher[] subSearchers; - if (ctx instanceof LeafReaderContext) { - subSearchers = new ShardSearcher[1]; - subSearchers[0] = new ShardSearcher((LeafReaderContext) ctx, ctx); - } else { - final CompositeReaderContext compCTX = (CompositeReaderContext) ctx; - final int size = compCTX.leaves().size(); - subSearchers = new ShardSearcher[size]; - for(int searcherIDX=0;searcherIDX aggs = new ArrayList<>(); Query rewritten = searcher.rewrite(query); - Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); - for (ShardSearcher subSearcher : subSearchers) { - MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C a = createAggregator(query, builder, subSearcher, indexSettings, shardBucketConsumer, fieldTypes); - a.preCollection(); - subSearcher.search(weight, a); - a.postCollection(); - InternalAggregation agg = a.buildTopLevel(); - aggs.add(agg); - } - if (aggs.isEmpty()) { - return (A) root.buildEmptyAggregation(); - } else { - if (randomBoolean() && aggs.size() > 1) { - // sometimes do an incremental reduce - int toReduceSize = aggs.size(); - Collections.shuffle(aggs, random()); - int r = randomIntBetween(1, toReduceSize); - List toReduce = aggs.subList(0, r); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction( - root.context().bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); - A reduced = (A) aggs.get(0).reduce(toReduce, context); - aggs = new ArrayList<>(aggs.subList(r, toReduceSize)); - aggs.add(reduced); + if (randomBoolean() && searcher.getIndexReader().leaves().size() > 0) { + assertThat(ctx, instanceOf(CompositeReaderContext.class)); + final CompositeReaderContext compCTX = (CompositeReaderContext) ctx; + final int size = compCTX.leaves().size(); + final ShardSearcher[] subSearchers = new ShardSearcher[size]; + for (int searcherIDX = 0; searcherIDX < subSearchers.length; searcherIDX++) { + final LeafReaderContext leave = compCTX.leaves().get(searcherIDX); + subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX); } - // now do the final reduce - MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( - root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); + for (ShardSearcher subSearcher : subSearchers) { + MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + C a = createAggregator(query, builder, subSearcher, indexSettings, shardBucketConsumer, fieldTypes); + a.preCollection(); + Weight weight = subSearcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); + subSearcher.search(weight, a); + a.postCollection(); + aggs.add(a.buildTopLevel()); + } + } else { + root.preCollection(); + searcher.search(rewritten, root); + root.postCollection(); + aggs.add(root.buildTopLevel()); + } + + if (randomBoolean() && aggs.size() > 1) { + // sometimes do an incremental reduce + int toReduceSize = aggs.size(); + Collections.shuffle(aggs, random()); + int r = randomIntBetween(1, toReduceSize); + List toReduce = aggs.subList(0, r); + InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction( + root.context().bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); + A reduced = (A) aggs.get(0).reduce(toReduce, context); + aggs = new ArrayList<>(aggs.subList(r, toReduceSize)); + aggs.add(reduced); + } + + // now do the final reduce + MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( + root.context().bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); - @SuppressWarnings("unchecked") - A internalAgg = (A) aggs.get(0).reduce(aggs, context); + @SuppressWarnings("unchecked") + A internalAgg = (A) aggs.get(0).reduce(aggs, context); - // materialize any parent pipelines - internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines); + // materialize any parent pipelines + internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines); - // materialize any sibling pipelines at top level - for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) { - internalAgg = (A) pipelineAggregator.reduce(internalAgg, context); - } - doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); - return internalAgg; + // materialize any sibling pipelines at top level + for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) { + internalAgg = (A) pipelineAggregator.reduce(internalAgg, context); } - + doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + return internalAgg; } protected void doAssertReducedMultiBucketConsumer(Aggregation agg, MultiBucketConsumerService.MultiBucketConsumer bucketConsumer) { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java index bddea56afb376..1675708122ef1 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentileRanksAggregatorTests.java @@ -94,7 +94,7 @@ public void testSimple() throws IOException { MappedFieldType fieldType = new HistogramFieldMapper.HistogramFieldType("field", true, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + PercentileRanks ranks = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index f594422d571a6..d46b75ad31f1d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -531,7 +531,7 @@ private InternalAggregation collect(AggregationBuilder builder, Query query, try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - return search(indexSearcher, query, builder, fields); + return searchAndReduce(indexSearcher, query, builder, fields); } } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 35eeece14f0cb..b603e1b40280e 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -654,7 +654,7 @@ protected void doNextSearch(SearchRequest request, ActionListener= 0 ? lon : Double.POSITIVE_INFINITY)); @@ -149,7 +149,7 @@ public void testInvalidMissing() throws Exception { try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); + () -> searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType)); assertThat(exception.getMessage(), startsWith("Unknown geometry type")); } } @@ -204,7 +204,7 @@ public void testRandomShapes() throws Exception { = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", true, true, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); - InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE));