[Discover] Show ignored field values (#115040) (#115584)

* WIP replacing indexPattern.flattenHit by tabify

* Fix jest tests

* Read metaFields from index pattern

* Remove old test code

* remove unnecessary changes

* Remove flattenHitWrapper APIs

* Fix imports

* Fix missing metaFields

* Add all meta fields to allowlist

* Improve inline comments

* Move flattenHit test to new implementation

* Add deprecation comment to implementation

* WIP - Show ignored field values

* Disable filters in doc_table

* remove redundant comments

* No, it wasn't

* start warning message

* Enable ignored values in CSV reports

* Add help tooltip

* Better styling with warning plus collapsible button

* Disable filtering within table for ignored values

* Fix jest tests

* Fix types in tests

* Add more tests and documentation

* Remove comment

* Move dangerouslySetInnerHTML into helper method

* Extract document formatting into common utility

* Remove HTML source field formatter

* Move formatHit to Discover

* Change wording of ignored warning

* Add cache for formatted hits

* Remove dead type

* Fix row_formatter for objects

* Improve mobile layout

* Fix jest tests

* Fix typo

* Remove additional span again

* Change mock to revert test

* Improve tests

* More jest tests

* Fix typo

* Change wording

* Remove dead comment

Co-authored-by: Kibana Machine <[email protected]>
# Conflicts:
#	src/plugins/data_views/public/index.ts

src/plugins/data/common/search/tabify/tabify_docs.test.ts

@@ -41,6 +41,11 @@ function create(id: string) {
  });
 }
 
+const meta = {
+ _index: 'index-name',
+ _id: '1',
+};
+
 describe('tabify_docs', () => {
  describe('flattenHit', () => {
  let indexPattern: DataView;
@@ -70,6 +75,50 @@ describe('tabify_docs', () => {
  expect(Object.keys(response)).toEqual(expectedOrder);
  expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
  });
+
+ it('does merge values from ignored_field_values and fields correctly', () => {
+ const flatten = flattenHit(
+ {
+ ...meta,
+ fields: { 'extension.keyword': ['foo'], extension: ['foo', 'ignored'] },
+ ignored_field_values: {
+ 'extension.keyword': ['ignored'],
+ fully_ignored: ['some', 'value'],
+ },
+ },
+ indexPattern,
+ { includeIgnoredValues: true }
+ );
+ expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
+ expect(flatten).toHaveProperty('extension', ['foo', 'ignored']);
+ expect(flatten).toHaveProperty('fully_ignored', ['some', 'value']);
+ });
+
+ it('does not merge values from ignored_field_values into _source', () => {
+ const flatten = flattenHit(
+ {
+ ...meta,
+ _source: { 'extension.keyword': ['foo', 'ignored'] },
+ ignored_field_values: { 'extension.keyword': ['ignored'] },
+ },
+ indexPattern,
+ { includeIgnoredValues: true, source: true }
+ );
+ expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
+ });
+
+ it('does merge ignored_field_values when no _source was present, even when parameter was on', () => {
+ const flatten = flattenHit(
+ {
+ ...meta,
+ fields: { 'extension.keyword': ['foo'] },
+ ignored_field_values: { 'extension.keyword': ['ignored'] },
+ },
+ indexPattern,
+ { includeIgnoredValues: true, source: true }
+ );
+ expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
+ });
  });
 
  describe('tabifyDocs', () => {

src/plugins/data/common/search/tabify/tabify_docs.ts

@@ -55,8 +55,18 @@ export interface TabifyDocsOptions {
  * merged into the flattened document.
  */
  source?: boolean;
+ /**
+ * If set to `true` values that have been ignored in ES (ignored_field_values)
+ * will be merged into the flattened document. This will only have an effect if
+ * the `hit` has been retrieved using the `fields` option.
+ */
+ includeIgnoredValues?: boolean;
 }
 
+// This is an overwrite of the SearchHit type to add the ignored_field_values.
+// Can be removed once the estypes.SearchHit knows about ignored_field_values
+type Hit<T = unknown> = estypes.SearchHit<T> & { ignored_field_values?: Record<string, unknown[]> };
+
 /**
  * Flattens an individual hit (from an ES response) into an object. This will
  * create flattened field names, like `user.name`.
@@ -65,11 +75,7 @@ export interface TabifyDocsOptions {
  * @param indexPattern The index pattern for the requested index if available.
  * @param params Parameters how to flatten the hit
  */
-export function flattenHit(
- hit: estypes.SearchHit,
- indexPattern?: IndexPattern,
- params?: TabifyDocsOptions
-) {
+export function flattenHit(hit: Hit, indexPattern?: IndexPattern, params?: TabifyDocsOptions) {
  const flat = {} as Record<string, any>;
 
  function flatten(obj: Record<string, any>, keyPrefix: string = '') {
@@ -109,6 +115,28 @@ export function flattenHit(
  flatten(hit.fields || {});
  if (params?.source !== false && hit._source) {
  flatten(hit._source as Record<string, any>);
+ } else if (params?.includeIgnoredValues && hit.ignored_field_values) {
+ // If enabled merge the ignored_field_values into the flattened hit. This will
+ // merge values that are not actually indexed by ES (i.e. ignored), e.g. because
+ // they were above the `ignore_above` limit or malformed for specific types.
+ // This API will only contain the values that were actually ignored, i.e. for the same
+ // field there might exist another value in the `fields` response, why this logic
+ // merged them both together. We do not merge this (even if enabled) in case source has been
+ // merged, since we would otherwise duplicate values, since ignore_field_values and _source
+ // contain the same values.
+ Object.entries(hit.ignored_field_values).forEach(([fieldName, fieldValue]) => {
+ if (flat[fieldName]) {
+ // If there was already a value from the fields API, make sure we're merging both together
+ if (Array.isArray(flat[fieldName])) {
+ flat[fieldName] = [...flat[fieldName], ...fieldValue];
+ } else {
+ flat[fieldName] = [flat[fieldName], ...fieldValue];
+ }
+ } else {
+ // If no previous value was assigned we can simply use the value from `ignored_field_values` as it is
+ flat[fieldName] = fieldValue;
+ }
+ });
  }
 
  // Merge all valid meta fields into the flattened object

src/plugins/data_views/common/data_views/data_view.ts

@@ -17,7 +17,6 @@ import { DuplicateField } from '../../../kibana_utils/common';
 
 import { IIndexPattern, IFieldType } from '../../common';
 import { DataViewField, IIndexPatternFieldList, fieldList } from '../fields';
-import { formatHitProvider } from './format_hit';
 import { flattenHitWrapper } from './flatten_hit';
 import {
  FieldFormatsStartCommon,
@@ -45,8 +44,6 @@ interface SavedObjectBody {
  type?: string;
 }
 
-type FormatFieldFn = (hit: Record<string, any>, fieldName: string) => any;
-
 export class DataView implements IIndexPattern {
  public id?: string;
  public title: string = '';
@@ -67,11 +64,6 @@ export class DataView implements IIndexPattern {
  * Type is used to identify rollup index patterns
  */
  public type: string | undefined;
- public formatHit: {
- (hit: Record<string, any>, type?: string): any;
- formatField: FormatFieldFn;
- };
- public formatField: FormatFieldFn;
  /**
  * @deprecated Use `flattenHit` utility method exported from data plugin instead.
  */
@@ -103,11 +95,6 @@ export class DataView implements IIndexPattern {
  this.fields = fieldList([], this.shortDotsEnable);
 
  this.flattenHit = flattenHitWrapper(this, metaFields);
- this.formatHit = formatHitProvider(
- this,
- fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING)
- );
- this.formatField = this.formatHit.formatField;
 
  // set values
  this.id = spec.id;

src/plugins/data_views/common/data_views/index.ts

@@ -8,6 +8,5 @@
 
 export * from './_pattern_cache';
 export * from './flatten_hit';
-export * from './format_hit';
 export * from './data_view';
 export * from './data_views';

src/plugins/data_views/public/index.ts

@@ -13,11 +13,7 @@ export {
  ILLEGAL_CHARACTERS,
  validateDataView,
 } from '../common/lib';
-export {
- formatHitProvider,
- onRedirectNoIndexPattern,
- onUnsupportedTimePattern,
-} from './data_views';
+export { onRedirectNoIndexPattern, onUnsupportedTimePattern } from './data_views';
 
 export { IndexPatternField, IIndexPatternFieldList, TypeMeta } from '../common';
 

src/plugins/discover/kibana.json

@@ -8,6 +8,7 @@
  "data",
  "embeddable",
  "inspector",
+ "fieldFormats",
  "kibanaLegacy",
  "urlForwarding",
  "navigation",
@@ -16,7 +17,7 @@
  "indexPatternFieldEditor"
  ],
  "optionalPlugins": ["home", "share", "usageCollection", "spaces"],
- "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "fieldFormats", "dataViews"],
+ "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "dataViews"],
  "extraPublicDirs": ["common"],
  "owner": {
  "name": "Data Discovery",

src/plugins/discover/public/__mocks__/index_pattern.ts

@@ -6,8 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { estypes } from '@elastic/elasticsearch';
-import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
+import { IIndexPatternFieldList } from '../../../data/common';
 import { IndexPattern } from '../../../data/common';
 
 const fields = [
@@ -28,33 +27,38 @@ const fields = [
  {
  name: 'message',
  type: 'string',
+ displayName: 'message',
  scripted: false,
  filterable: false,
  aggregatable: false,
  },
  {
  name: 'extension',
  type: 'string',
+ displayName: 'extension',
  scripted: false,
  filterable: true,
  aggregatable: true,
  },
  {
  name: 'bytes',
  type: 'number',
+ displayName: 'bytesDisplayName',
  scripted: false,
  filterable: true,
  aggregatable: true,
  },
  {
  name: 'scripted',
  type: 'number',
+ displayName: 'scripted',
  scripted: true,
  filterable: false,
  },
  {
  name: 'object.value',
  type: 'number',
+ displayName: 'object.value',
  scripted: false,
  filterable: true,
  aggregatable: true,
@@ -73,23 +77,15 @@ const indexPattern = {
  id: 'the-index-pattern-id',
  title: 'the-index-pattern-title',
  metaFields: ['_index', '_score'],
- formatField: jest.fn(),
- flattenHit: undefined,
- formatHit: jest.fn((hit) => (hit.fields ? hit.fields : hit._source)),
  fields,
  getComputedFields: () => ({ docvalueFields: [], scriptFields: {}, storedFields: ['*'] }),
  getSourceFiltering: () => ({}),
  getFieldByName: jest.fn(() => ({})),
  timeFieldName: '',
  docvalueFields: [],
- getFormatterForField: () => ({ convert: () => 'formatted' }),
+ getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
 } as unknown as IndexPattern;
 
 indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
-indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
- return fieldName === '_source'
- ? hit._source
- : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
-};
 
 export const indexPatternMock = indexPattern;

src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts

@@ -6,9 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
+import { IIndexPatternFieldList } from '../../../data/common';
 import { IndexPattern } from '../../../data/common';
-import type { estypes } from '@elastic/elasticsearch';
 
 const fields = [
  {
@@ -64,23 +63,16 @@ const indexPattern = {
  id: 'index-pattern-with-timefield-id',
  title: 'index-pattern-with-timefield',
  metaFields: ['_index', '_score'],
- flattenHit: undefined,
- formatHit: jest.fn((hit) => hit._source),
  fields,
  getComputedFields: () => ({}),
  getSourceFiltering: () => ({}),
  getFieldByName: (name: string) => fields.getByName(name),
  timeFieldName: 'timestamp',
- getFormatterForField: () => ({ convert: () => 'formatted' }),
+ getFormatterForField: () => ({ convert: (value: unknown) => value }),
  isTimeNanosBased: () => false,
  popularizeField: () => {},
 } as unknown as IndexPattern;
 
 indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
-indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
- return fieldName === '_source'
- ? hit._source
- : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
-};
 
 export const indexPatternWithTimefieldMock = indexPattern;