[Lens] Expose Elasticsearch accuracy warnings to the user (elastic#11…

…6632) * [Lens] Expose Elasticsearch accuracy warnings to the user Closes: elastic#94918 * fix comments * update text Co-authored-by: Kibana Machine <[email protected]>
claudiopro · Nov 11, 2021 · 6507038 · 6507038
1 parent 6c757a1
commit 6507038
Show file tree

Hide file tree

Showing 60 changed files with 293 additions and 59 deletions.
diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md
@@ -89,6 +89,7 @@ readonly links: {
  readonly range: string;
  readonly significant_terms: string;
  readonly terms: string;
+ readonly terms_doc_count_error: string;
  readonly avg: string;
  readonly avg_bucket: string;
  readonly max_bucket: string;

diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md
diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts
@@ -118,6 +118,7 @@ export class DocLinksService {
  range: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-range-aggregation.html`,
  significant_terms: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-significantterms-aggregation.html`,
  terms: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-terms-aggregation.html`,
+ terms_doc_count_error: `${ELASTICSEARCH_DOCS}search-aggregations-bucket-terms-aggregation.html#_per_bucket_document_count_error`,
  avg: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-avg-aggregation.html`,
  avg_bucket: `${ELASTICSEARCH_DOCS}search-aggregations-pipeline-avg-bucket-aggregation.html`,
  max_bucket: `${ELASTICSEARCH_DOCS}search-aggregations-pipeline-max-bucket-aggregation.html`,
@@ -613,6 +614,7 @@ export interface DocLinksStart {
  readonly range: string;
  readonly significant_terms: string;
  readonly terms: string;
+ readonly terms_doc_count_error: string;
  readonly avg: string;
  readonly avg_bucket: string;
  readonly max_bucket: string;

diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md
@@ -557,6 +557,7 @@ export interface DocLinksStart {
  readonly range: string;
  readonly significant_terms: string;
  readonly terms: string;
+ readonly terms_doc_count_error: string;
  readonly avg: string;
  readonly avg_bucket: string;
  readonly max_bucket: string;

diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts
@@ -53,6 +53,7 @@ export interface AggTypeConfig<
  json?: boolean;
  decorateAggConfig?: () => any;
  postFlightRequest?: PostFlightRequestFn<TAggConfig>;
+ hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
  getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat;
  getValue?: (agg: TAggConfig, bucket: any) => any;
  getKey?: (bucket: any, key: any, agg: TAggConfig) => any;
@@ -180,6 +181,9 @@ export class AggType<
  * is created, giving the agg type a chance to modify the agg config
  */
  decorateAggConfig: () => any;
+
+ hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
+
  /**
  * A function that needs to be called after the main request has been made
  * and should return an updated response
@@ -283,6 +287,7 @@ export class AggType<
  this.getResponseAggs = config.getResponseAggs || (() => {});
  this.decorateAggConfig = config.decorateAggConfig || (() => ({}));
  this.postFlightRequest = config.postFlightRequest || identity;
+ this.hasPrecisionError = config.hasPrecisionError;
 
  this.getSerializedFormat =
  config.getSerializedFormat ||

diff --git a/src/plugins/data/common/search/aggs/buckets/terms.test.ts b/src/plugins/data/common/search/aggs/buckets/terms.test.ts
@@ -286,7 +286,19 @@ describe('Terms Agg', () => {
  { typesRegistry: mockAggTypesRegistry() }
  );
  const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+
  expect(params.order).toEqual({ 'test-orderAgg.50': 'desc' });
  });
+
+ test('should override "hasPrecisionError" for the "terms" bucket type', () => {
+ const aggConfigs = getAggConfigs();
+ const { type } = aggConfigs.aggs[0];
+
+ expect(type.hasPrecisionError).toBeInstanceOf(Function);
+
+ expect(type.hasPrecisionError!({})).toBeFalsy();
+ expect(type.hasPrecisionError!({ doc_count_error_upper_bound: 0 })).toBeFalsy();
+ expect(type.hasPrecisionError!({ doc_count_error_upper_bound: -1 })).toBeTruthy();
+ });
  });
 });
diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts
@@ -85,6 +85,7 @@ export const getTermsBucketAgg = () =>
  };
  },
  createFilter: createFilterTerms,
+ hasPrecisionError: (aggBucket) => Boolean(aggBucket?.doc_count_error_upper_bound),
  postFlightRequest: async (
  resp,
  aggConfigs,

diff --git a/src/plugins/data/common/search/tabify/index.ts b/src/plugins/data/common/search/tabify/index.ts
@@ -9,3 +9,4 @@
 export { tabifyDocs, flattenHit } from './tabify_docs';
 export { tabifyAggResponse } from './tabify';
 export { tabifyGetColumns } from './get_columns';
+export { checkColumnForPrecisionError } from './utils';
diff --git a/src/plugins/data/common/search/tabify/response_writer.test.ts b/src/plugins/data/common/search/tabify/response_writer.test.ts
@@ -166,6 +166,7 @@ describe('TabbedAggResponseWriter class', () => {
  field: 'geo.src',
  source: 'esaggs',
  sourceParams: {
+ hasPrecisionError: false,
  enabled: true,
  id: '1',
  indexPatternId: '1234',
@@ -193,6 +194,7 @@ describe('TabbedAggResponseWriter class', () => {
  },
  source: 'esaggs',
  sourceParams: {
+ hasPrecisionError: false,
  appliedTimeRange: undefined,
  enabled: true,
  id: '2',
@@ -227,6 +229,7 @@ describe('TabbedAggResponseWriter class', () => {
  field: 'geo.src',
  source: 'esaggs',
  sourceParams: {
+ hasPrecisionError: false,
  enabled: true,
  id: '1',
  indexPatternId: '1234',
@@ -254,6 +257,7 @@ describe('TabbedAggResponseWriter class', () => {
  },
  source: 'esaggs',
  sourceParams: {
+ hasPrecisionError: false,
  appliedTimeRange: undefined,
  enabled: true,
  id: '2',

diff --git a/src/plugins/data/common/search/tabify/response_writer.ts b/src/plugins/data/common/search/tabify/response_writer.ts
@@ -10,7 +10,7 @@ import { isEmpty } from 'lodash';
 import { IAggConfigs } from '../aggs';
 import { tabifyGetColumns } from './get_columns';
 
-import { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
+import type { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
 import { Datatable, DatatableColumn } from '../../../../expressions/common/expression_types/specs';
 
 interface BufferColumn {
@@ -80,6 +80,7 @@ export class TabbedAggResponseWriter {
  params: column.aggConfig.toSerializedFieldFormat(),
  source: 'esaggs',
  sourceParams: {
+ hasPrecisionError: Boolean(column.hasPrecisionError),
  indexPatternId: column.aggConfig.getIndexPattern()?.id,
  appliedTimeRange:
  column.aggConfig.params.field?.name &&

diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts
@@ -42,8 +42,14 @@ export function tabifyAggResponse(
 
  switch (agg.type.type) {
  case AggGroupNames.Buckets:
- const aggBucket = get(bucket, agg.id);
+ const aggBucket = get(bucket, agg.id) as Record<string, unknown>;
  const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, respOpts?.timeRange);
+ const precisionError = agg.type.hasPrecisionError?.(aggBucket);
+
+ if (precisionError) {
+ // "сolumn" mutation, we have to do this here as this value is filled in based on aggBucket value
+ column.hasPrecisionError = true;
+ }
 
  if (tabifyBuckets.length) {
  tabifyBuckets.forEach((subBucket, tabifyBucketKey) => {

diff --git a/src/plugins/data/common/search/tabify/types.ts b/src/plugins/data/common/search/tabify/types.ts
@@ -41,6 +41,7 @@ export interface TabbedAggColumn {
  aggConfig: IAggConfig;
  id: string;
  name: string;
+ hasPrecisionError?: boolean;
 }
 
 /** @public **/

diff --git a/src/plugins/data/common/search/tabify/utils.test.ts b/src/plugins/data/common/search/tabify/utils.test.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { checkColumnForPrecisionError } from './utils';
+import type { DatatableColumn } from '../../../../expressions';
+
+describe('tabify utils', () => {
+ describe('checkDatatableForPrecisionError', () => {
+ test('should return true if there is a precision error in the column', () => {
+ expect(
+ checkColumnForPrecisionError({
+ meta: {
+ sourceParams: {
+ hasPrecisionError: true,
+ },
+ },
+ } as unknown as DatatableColumn)
+ ).toBeTruthy();
+ });
+ test('should return false if there is no precision error in the column', () => {
+ expect(
+ checkColumnForPrecisionError({
+ meta: {
+ sourceParams: {
+ hasPrecisionError: false,
+ },
+ },
+ } as unknown as DatatableColumn)
+ ).toBeFalsy();
+ });
+ test('should return false if precision error is not defined', () => {
+ expect(
+ checkColumnForPrecisionError({
+ meta: {
+ sourceParams: {},
+ },
+ } as unknown as DatatableColumn)
+ ).toBeFalsy();
+ });
+ });
+});
diff --git a/src/plugins/data/common/search/tabify/utils.ts b/src/plugins/data/common/search/tabify/utils.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { DatatableColumn } from '../../../../expressions';
+
+/** @public **/
+export const checkColumnForPrecisionError = (column: DatatableColumn) =>
+ column.meta.sourceParams?.hasPrecisionError;
diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts
@@ -139,6 +139,7 @@ import {
  // tabify
  tabifyAggResponse,
  tabifyGetColumns,
+ checkColumnForPrecisionError,
 } from '../common';
 
 export { AggGroupLabels, AggGroupNames, METRIC_TYPES, BUCKET_TYPES } from '../common';
@@ -246,6 +247,7 @@ export const search = {
  getResponseInspectorStats,
  tabifyAggResponse,
  tabifyGetColumns,
+ checkColumnForPrecisionError,
 };
 
 /*

diff --git a/src/plugins/data/public/search/index.ts b/src/plugins/data/public/search/index.ts
@@ -36,6 +36,7 @@ export {
  parseSearchSourceJSON,
  SearchSource,
  SortDirection,
+ checkColumnForPrecisionError,
 } from '../../common/search';
 export type {
  ISessionService,

diff --git a/test/interpreter_functional/snapshots/baseline/combined_test2.json b/test/interpreter_functional/snapshots/baseline/combined_test2.json
@@ -1 +1 @@
-{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}
+{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}
diff --git a/test/interpreter_functional/snapshots/baseline/combined_test3.json b/test/interpreter_functional/snapshots/baseline/combined_test3.json
@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
diff --git a/test/interpreter_functional/snapshots/baseline/final_output_test.json b/test/interpreter_functional/snapshots/baseline/final_output_test.json
@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"labels":{"show":true},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}