Skip to content

Commit

Permalink
fix stats aggregator tests
Browse files Browse the repository at this point in the history
With #60683 we stopped forcing aggregating all docs using a single
Aggregator which made some of our accuracy assumptions about the stats
aggregator incorrect. This adds a test that does the forcing and asserts
the old accuracy and adds a test without the forcing with much looser
accuracy guarantees.

Closes #61132
  • Loading branch information
nik9000 committed Aug 19, 2020
1 parent b1aa0d8 commit 70128e0
Showing 1 changed file with 34 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
Expand Down Expand Up @@ -140,7 +141,7 @@ public void testRandomLongs() throws IOException {
public void testSummationAccuracy() throws IOException {
// Summing up a normal array and expect an accurate value
double[] values = new double[]{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7};
verifySummationOfDoubles(values, 15.3, 0.9, 0d);
verifySummationOfDoubles(values, 15.3, 0.9, 0d, values.length * TOLERANCE);

// Summing up an array which contains NaN and infinities and expect a result same as naive summation
int n = randomIntBetween(5, 10);
Expand All @@ -152,24 +153,29 @@ public void testSummationAccuracy() throws IOException {
: randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true);
sum += values[i];
}
verifySummationOfDoubles(values, sum, sum / n, TOLERANCE);
verifySummationOfDoubles(values, sum, sum / n, TOLERANCE, n * TOLERANCE);

// Summing up some big double values and expect infinity result
n = randomIntBetween(5, 10);
double[] largeValues = new double[n];
for (int i = 0; i < n; i++) {
largeValues[i] = Double.MAX_VALUE;
}
verifySummationOfDoubles(largeValues, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, 0d);
verifySummationOfDoubles(largeValues, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, 0d, 0d);

for (int i = 0; i < n; i++) {
largeValues[i] = -Double.MAX_VALUE;
}
verifySummationOfDoubles(largeValues, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY, 0d);
verifySummationOfDoubles(largeValues, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY, 0d, 0d);
}

private void verifySummationOfDoubles(double[] values, double expectedSum,
double expectedAvg, double delta) throws IOException {
private void verifySummationOfDoubles(
double[] values,
double expectedSum,
double expectedAvg,
double singleSegmentDelta,
double manySegmentDelta
) throws IOException {
MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.DOUBLE);

double max = Double.NEGATIVE_INFINITY;
Expand All @@ -183,14 +189,33 @@ private void verifySummationOfDoubles(double[] values, double expectedSum,
testCase(
stats("_name").field(ft.name()),
iw -> {
List<List<NumericDocValuesField>> docs = new ArrayList<>();
for (double value : values) {
iw.addDocument(singleton(new NumericDocValuesField(ft.name(), NumericUtils.doubleToSortableLong(value))));
docs.add(singletonList(new NumericDocValuesField(ft.name(), NumericUtils.doubleToSortableLong(value))));
}
iw.addDocuments(docs);
},
stats -> {
assertEquals(values.length, stats.getCount());
assertEquals(expectedAvg, stats.getAvg(), delta);
assertEquals(expectedSum, stats.getSum(), delta);
assertEquals(expectedAvg, stats.getAvg(), singleSegmentDelta);
assertEquals(expectedSum, stats.getSum(), singleSegmentDelta);
assertEquals(expectedMax, stats.getMax(), 0d);
assertEquals(expectedMin, stats.getMin(), 0d);
assertTrue(AggregationInspectionHelper.hasValue(stats));
},
singleton(ft)
);
testCase(
stats("_name").field(ft.name()),
iw -> {
for (double value : values) {
iw.addDocument(singletonList(new NumericDocValuesField(ft.name(), NumericUtils.doubleToSortableLong(value))));
}
},
stats -> {
assertEquals(values.length, stats.getCount());
assertEquals(expectedAvg, stats.getAvg(), manySegmentDelta);
assertEquals(expectedSum, stats.getSum(), manySegmentDelta);
assertEquals(expectedMax, stats.getMax(), 0d);
assertEquals(expectedMin, stats.getMin(), 0d);
assertTrue(AggregationInspectionHelper.hasValue(stats));
Expand Down

0 comments on commit 70128e0

Please sign in to comment.