diff --git a/src/plugins/data/common/search/tabify/tabify_docs.test.ts b/src/plugins/data/common/search/tabify/tabify_docs.test.ts index a2910a1be4a9a4..1964247b09585a 100644 --- a/src/plugins/data/common/search/tabify/tabify_docs.test.ts +++ b/src/plugins/data/common/search/tabify/tabify_docs.test.ts @@ -41,6 +41,11 @@ function create(id: string) { }); } +const meta = { + _index: 'index-name', + _id: '1', +}; + describe('tabify_docs', () => { describe('flattenHit', () => { let indexPattern: DataView; @@ -70,6 +75,50 @@ describe('tabify_docs', () => { expect(Object.keys(response)).toEqual(expectedOrder); expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder); }); + + it('does merge values from ignored_field_values and fields correctly', () => { + const flatten = flattenHit( + { + ...meta, + fields: { 'extension.keyword': ['foo'], extension: ['foo', 'ignored'] }, + ignored_field_values: { + 'extension.keyword': ['ignored'], + fully_ignored: ['some', 'value'], + }, + }, + indexPattern, + { includeIgnoredValues: true } + ); + expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']); + expect(flatten).toHaveProperty('extension', ['foo', 'ignored']); + expect(flatten).toHaveProperty('fully_ignored', ['some', 'value']); + }); + + it('does not merge values from ignored_field_values into _source', () => { + const flatten = flattenHit( + { + ...meta, + _source: { 'extension.keyword': ['foo', 'ignored'] }, + ignored_field_values: { 'extension.keyword': ['ignored'] }, + }, + indexPattern, + { includeIgnoredValues: true, source: true } + ); + expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']); + }); + + it('does merge ignored_field_values when no _source was present, even when parameter was on', () => { + const flatten = flattenHit( + { + ...meta, + fields: { 'extension.keyword': ['foo'] }, + ignored_field_values: { 'extension.keyword': ['ignored'] }, + }, + indexPattern, + { includeIgnoredValues: true, source: true } + ); + expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']); + }); }); describe('tabifyDocs', () => { diff --git a/src/plugins/data/common/search/tabify/tabify_docs.ts b/src/plugins/data/common/search/tabify/tabify_docs.ts index 4259488771761c..353a0c10ba12aa 100644 --- a/src/plugins/data/common/search/tabify/tabify_docs.ts +++ b/src/plugins/data/common/search/tabify/tabify_docs.ts @@ -55,8 +55,18 @@ export interface TabifyDocsOptions { * merged into the flattened document. */ source?: boolean; + /** + * If set to `true` values that have been ignored in ES (ignored_field_values) + * will be merged into the flattened document. This will only have an effect if + * the `hit` has been retrieved using the `fields` option. + */ + includeIgnoredValues?: boolean; } +// This is an overwrite of the SearchHit type to add the ignored_field_values. +// Can be removed once the estypes.SearchHit knows about ignored_field_values +type Hit = estypes.SearchHit & { ignored_field_values?: Record }; + /** * Flattens an individual hit (from an ES response) into an object. This will * create flattened field names, like `user.name`. @@ -65,11 +75,7 @@ export interface TabifyDocsOptions { * @param indexPattern The index pattern for the requested index if available. * @param params Parameters how to flatten the hit */ -export function flattenHit( - hit: estypes.SearchHit, - indexPattern?: IndexPattern, - params?: TabifyDocsOptions -) { +export function flattenHit(hit: Hit, indexPattern?: IndexPattern, params?: TabifyDocsOptions) { const flat = {} as Record; function flatten(obj: Record, keyPrefix: string = '') { @@ -109,6 +115,28 @@ export function flattenHit( flatten(hit.fields || {}); if (params?.source !== false && hit._source) { flatten(hit._source as Record); + } else if (params?.includeIgnoredValues && hit.ignored_field_values) { + // If enabled merge the ignored_field_values into the flattened hit. This will + // merge values that are not actually indexed by ES (i.e. ignored), e.g. because + // they were above the `ignore_above` limit or malformed for specific types. + // This API will only contain the values that were actually ignored, i.e. for the same + // field there might exist another value in the `fields` response, why this logic + // merged them both together. We do not merge this (even if enabled) in case source has been + // merged, since we would otherwise duplicate values, since ignore_field_values and _source + // contain the same values. + Object.entries(hit.ignored_field_values).forEach(([fieldName, fieldValue]) => { + if (flat[fieldName]) { + // If there was already a value from the fields API, make sure we're merging both together + if (Array.isArray(flat[fieldName])) { + flat[fieldName] = [...flat[fieldName], ...fieldValue]; + } else { + flat[fieldName] = [flat[fieldName], ...fieldValue]; + } + } else { + // If no previous value was assigned we can simply use the value from `ignored_field_values` as it is + flat[fieldName] = fieldValue; + } + }); } // Merge all valid meta fields into the flattened object diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index a1b6d4d731bcaa..e6533a8f9fb1e0 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -17,7 +17,6 @@ import { DuplicateField } from '../../../kibana_utils/common'; import { IIndexPattern, IFieldType } from '../../common'; import { DataViewField, IIndexPatternFieldList, fieldList } from '../fields'; -import { formatHitProvider } from './format_hit'; import { flattenHitWrapper } from './flatten_hit'; import { FieldFormatsStartCommon, @@ -45,8 +44,6 @@ interface SavedObjectBody { type?: string; } -type FormatFieldFn = (hit: Record, fieldName: string) => any; - export class DataView implements IIndexPattern { public id?: string; public title: string = ''; @@ -67,11 +64,6 @@ export class DataView implements IIndexPattern { * Type is used to identify rollup index patterns */ public type: string | undefined; - public formatHit: { - (hit: Record, type?: string): any; - formatField: FormatFieldFn; - }; - public formatField: FormatFieldFn; /** * @deprecated Use `flattenHit` utility method exported from data plugin instead. */ @@ -103,11 +95,6 @@ export class DataView implements IIndexPattern { this.fields = fieldList([], this.shortDotsEnable); this.flattenHit = flattenHitWrapper(this, metaFields); - this.formatHit = formatHitProvider( - this, - fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING) - ); - this.formatField = this.formatHit.formatField; // set values this.id = spec.id; diff --git a/src/plugins/data_views/common/data_views/format_hit.ts b/src/plugins/data_views/common/data_views/format_hit.ts deleted file mode 100644 index c8e6e8e337155b..00000000000000 --- a/src/plugins/data_views/common/data_views/format_hit.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import _ from 'lodash'; -import { DataView } from './data_view'; -import { FieldFormatsContentType } from '../../../field_formats/common'; - -const formattedCache = new WeakMap(); -const partialFormattedCache = new WeakMap(); - -// Takes a hit, merges it with any stored/scripted fields, and with the metaFields -// returns a formatted version -export function formatHitProvider(dataView: DataView, defaultFormat: any) { - function convert( - hit: Record, - val: any, - fieldName: string, - type: FieldFormatsContentType = 'html' - ) { - const field = dataView.fields.getByName(fieldName); - const format = field ? dataView.getFormatterForField(field) : defaultFormat; - - return format.convert(val, type, { field, hit, indexPattern: dataView }); - } - - function formatHit(hit: Record, type: string = 'html') { - const cached = formattedCache.get(hit); - if (cached) { - return cached; - } - - // use and update the partial cache, but don't rewrite it. - // _source is stored in partialFormattedCache but not formattedCache - const partials = partialFormattedCache.get(hit) || {}; - partialFormattedCache.set(hit, partials); - - const cache: Record = {}; - formattedCache.set(hit, cache); - - _.forOwn(dataView.flattenHit(hit), function (val: any, fieldName?: string) { - // sync the formatted and partial cache - if (!fieldName) { - return; - } - const formatted = - partials[fieldName] == null ? convert(hit, val, fieldName) : partials[fieldName]; - cache[fieldName] = partials[fieldName] = formatted; - }); - - return cache; - } - - formatHit.formatField = function (hit: Record, fieldName: string) { - let partials = partialFormattedCache.get(hit); - if (partials && partials[fieldName] != null) { - return partials[fieldName]; - } - - if (!partials) { - partials = {}; - partialFormattedCache.set(hit, partials); - } - - const val = fieldName === '_source' ? hit._source : dataView.flattenHit(hit)[fieldName]; - return convert(hit, val, fieldName); - }; - - return formatHit; -} diff --git a/src/plugins/data_views/common/data_views/index.ts b/src/plugins/data_views/common/data_views/index.ts index 7c94dff961c9c0..d925d42fbea0d5 100644 --- a/src/plugins/data_views/common/data_views/index.ts +++ b/src/plugins/data_views/common/data_views/index.ts @@ -8,6 +8,5 @@ export * from './_pattern_cache'; export * from './flatten_hit'; -export * from './format_hit'; export * from './data_view'; export * from './data_views'; diff --git a/src/plugins/data_views/public/index.ts b/src/plugins/data_views/public/index.ts index d8814e886fb0e8..b220aea6f392c6 100644 --- a/src/plugins/data_views/public/index.ts +++ b/src/plugins/data_views/public/index.ts @@ -13,11 +13,7 @@ export { ILLEGAL_CHARACTERS, validateDataView, } from '../common/lib'; -export { - formatHitProvider, - onRedirectNoIndexPattern, - onUnsupportedTimePattern, -} from './data_views'; +export { onRedirectNoIndexPattern, onUnsupportedTimePattern } from './data_views'; export { IndexPatternField, IIndexPatternFieldList, TypeMeta } from '../common'; diff --git a/src/plugins/discover/kibana.json b/src/plugins/discover/kibana.json index 3d5fdefd276d34..791ce54a0cb1b3 100644 --- a/src/plugins/discover/kibana.json +++ b/src/plugins/discover/kibana.json @@ -8,6 +8,7 @@ "data", "embeddable", "inspector", + "fieldFormats", "kibanaLegacy", "urlForwarding", "navigation", @@ -16,7 +17,7 @@ "indexPatternFieldEditor" ], "optionalPlugins": ["home", "share", "usageCollection", "spaces"], - "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "fieldFormats", "dataViews"], + "requiredBundles": ["kibanaUtils", "home", "kibanaReact", "dataViews"], "extraPublicDirs": ["common"], "owner": { "name": "Data Discovery", diff --git a/src/plugins/discover/public/__mocks__/index_pattern.ts b/src/plugins/discover/public/__mocks__/index_pattern.ts index 2acb512617a6b4..d33445baa0a2b4 100644 --- a/src/plugins/discover/public/__mocks__/index_pattern.ts +++ b/src/plugins/discover/public/__mocks__/index_pattern.ts @@ -6,8 +6,7 @@ * Side Public License, v 1. */ -import type { estypes } from '@elastic/elasticsearch'; -import { flattenHit, IIndexPatternFieldList } from '../../../data/common'; +import { IIndexPatternFieldList } from '../../../data/common'; import { IndexPattern } from '../../../data/common'; const fields = [ @@ -28,6 +27,7 @@ const fields = [ { name: 'message', type: 'string', + displayName: 'message', scripted: false, filterable: false, aggregatable: false, @@ -35,6 +35,7 @@ const fields = [ { name: 'extension', type: 'string', + displayName: 'extension', scripted: false, filterable: true, aggregatable: true, @@ -42,6 +43,7 @@ const fields = [ { name: 'bytes', type: 'number', + displayName: 'bytesDisplayName', scripted: false, filterable: true, aggregatable: true, @@ -49,12 +51,14 @@ const fields = [ { name: 'scripted', type: 'number', + displayName: 'scripted', scripted: true, filterable: false, }, { name: 'object.value', type: 'number', + displayName: 'object.value', scripted: false, filterable: true, aggregatable: true, @@ -73,23 +77,15 @@ const indexPattern = { id: 'the-index-pattern-id', title: 'the-index-pattern-title', metaFields: ['_index', '_score'], - formatField: jest.fn(), - flattenHit: undefined, - formatHit: jest.fn((hit) => (hit.fields ? hit.fields : hit._source)), fields, getComputedFields: () => ({ docvalueFields: [], scriptFields: {}, storedFields: ['*'] }), getSourceFiltering: () => ({}), getFieldByName: jest.fn(() => ({})), timeFieldName: '', docvalueFields: [], - getFormatterForField: () => ({ convert: () => 'formatted' }), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), } as unknown as IndexPattern; indexPattern.isTimeBased = () => !!indexPattern.timeFieldName; -indexPattern.formatField = (hit: Record, fieldName: string) => { - return fieldName === '_source' - ? hit._source - : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName]; -}; export const indexPatternMock = indexPattern; diff --git a/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts b/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts index 6cf8e8b3485ff9..906ebdebdd06a6 100644 --- a/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts +++ b/src/plugins/discover/public/__mocks__/index_pattern_with_timefield.ts @@ -6,9 +6,8 @@ * Side Public License, v 1. */ -import { flattenHit, IIndexPatternFieldList } from '../../../data/common'; +import { IIndexPatternFieldList } from '../../../data/common'; import { IndexPattern } from '../../../data/common'; -import type { estypes } from '@elastic/elasticsearch'; const fields = [ { @@ -64,23 +63,16 @@ const indexPattern = { id: 'index-pattern-with-timefield-id', title: 'index-pattern-with-timefield', metaFields: ['_index', '_score'], - flattenHit: undefined, - formatHit: jest.fn((hit) => hit._source), fields, getComputedFields: () => ({}), getSourceFiltering: () => ({}), getFieldByName: (name: string) => fields.getByName(name), timeFieldName: 'timestamp', - getFormatterForField: () => ({ convert: () => 'formatted' }), + getFormatterForField: () => ({ convert: (value: unknown) => value }), isTimeNanosBased: () => false, popularizeField: () => {}, } as unknown as IndexPattern; indexPattern.isTimeBased = () => !!indexPattern.timeFieldName; -indexPattern.formatField = (hit: Record, fieldName: string) => { - return fieldName === '_source' - ? hit._source - : flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName]; -}; export const indexPatternWithTimefieldMock = indexPattern; diff --git a/src/plugins/discover/public/__mocks__/services.ts b/src/plugins/discover/public/__mocks__/services.ts index 8cc5ccf5aa121e..6a90ed42417e6f 100644 --- a/src/plugins/discover/public/__mocks__/services.ts +++ b/src/plugins/discover/public/__mocks__/services.ts @@ -13,6 +13,7 @@ import { CONTEXT_STEP_SETTING, DEFAULT_COLUMNS_SETTING, DOC_HIDE_TIME_COLUMN_SETTING, + MAX_DOC_FIELDS_DISPLAYED, SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING, } from '../../common'; @@ -43,9 +44,13 @@ export const discoverServiceMock = { save: true, }, }, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, filterManager: dataPlugin.query.filterManager, uiSettings: { - get: (key: string) => { + get: jest.fn((key: string) => { if (key === 'fields:popularLimit') { return 5; } else if (key === DEFAULT_COLUMNS_SETTING) { @@ -62,8 +67,10 @@ export const discoverServiceMock = { return false; } else if (key === SAMPLE_SIZE_SETTING) { return 250; + } else if (key === MAX_DOC_FIELDS_DISPLAYED) { + return 50; } - }, + }), isDefault: (key: string) => { return true; }, diff --git a/src/plugins/discover/public/application/apps/context/context_app.test.tsx b/src/plugins/discover/public/application/apps/context/context_app.test.tsx index 0e50f8f714a2c9..d1c557f2839bca 100644 --- a/src/plugins/discover/public/application/apps/context/context_app.test.tsx +++ b/src/plugins/discover/public/application/apps/context/context_app.test.tsx @@ -62,6 +62,10 @@ describe('ContextApp test', () => { navigation: mockNavigationPlugin, core: { notifications: { toasts: [] } }, history: () => {}, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, filterManager: mockFilterManager, uiSettings: uiSettingsMock, } as unknown as DiscoverServices); diff --git a/src/plugins/discover/public/application/apps/main/components/doc_table/components/table_row.tsx b/src/plugins/discover/public/application/apps/main/components/doc_table/components/table_row.tsx index d91735460af085..0bf4a36555d169 100644 --- a/src/plugins/discover/public/application/apps/main/components/doc_table/components/table_row.tsx +++ b/src/plugins/discover/public/application/apps/main/components/doc_table/components/table_row.tsx @@ -10,6 +10,7 @@ import React, { Fragment, useCallback, useMemo, useState } from 'react'; import classNames from 'classnames'; import { i18n } from '@kbn/i18n'; import { EuiButtonEmpty, EuiIcon } from '@elastic/eui'; +import { formatFieldValue } from '../../../../../helpers/format_value'; import { flattenHit } from '../../../../../../../../data/common'; import { DocViewer } from '../../../../../components/doc_viewer/doc_viewer'; import { FilterManager, IndexPattern } from '../../../../../../../../data/public'; @@ -58,7 +59,10 @@ export const TableRow = ({ }); const anchorDocTableRowSubj = row.isAnchor ? ' docTableAnchorRow' : ''; - const flattenedRow = useMemo(() => flattenHit(row, indexPattern), [indexPattern, row]); + const flattenedRow = useMemo( + () => flattenHit(row, indexPattern, { includeIgnoredValues: true }), + [indexPattern, row] + ); const mapping = useMemo(() => indexPattern.fields.getByName, [indexPattern]); // toggle display of the rows details, a full list of the fields from each row @@ -68,13 +72,24 @@ export const TableRow = ({ * Fill an element with the value of a field */ const displayField = (fieldName: string) => { - const formattedField = indexPattern.formatField(row, fieldName); - - // field formatters take care of escaping - // eslint-disable-next-line react/no-danger - const fieldElement = ; + // If we're formatting the _source column, don't use the regular field formatter, + // but our Discover mechanism to format a hit in a better human-readable way. + if (fieldName === '_source') { + return formatRow(row, indexPattern, fieldsToShow); + } + + const formattedField = formatFieldValue( + flattenedRow[fieldName], + row, + indexPattern, + mapping(fieldName) + ); - return
{fieldElement}
; + return ( + // formatFieldValue always returns sanitized HTML + // eslint-disable-next-line react/no-danger +
+ ); }; const inlineFilter = useCallback( (column: string, type: '+' | '-') => { @@ -141,10 +156,9 @@ export const TableRow = ({ ); } else { columns.forEach(function (column: string) { - // when useNewFieldsApi is true, addressing to the fields property is safe - if (useNewFieldsApi && !mapping(column) && !row.fields![column]) { + if (useNewFieldsApi && !mapping(column) && row.fields && !row.fields[column]) { const innerColumns = Object.fromEntries( - Object.entries(row.fields!).filter(([key]) => { + Object.entries(row.fields).filter(([key]) => { return key.indexOf(`${column}.`) === 0; }) ); @@ -161,7 +175,13 @@ export const TableRow = ({ /> ); } else { - const isFilterable = Boolean(mapping(column)?.filterable && filter); + // Check whether the field is defined as filterable in the mapping and does + // NOT have ignored values in it to determine whether we want to allow filtering. + // We should improve this and show a helpful tooltip why the filter buttons are not + // there/disabled when there are ignored values. + const isFilterable = Boolean( + mapping(column)?.filterable && filter && !row._ignored?.includes(column) + ); rowCells.push( { const hit = { _id: 'a', + _index: 'foo', _type: 'doc', _score: 1, _source: { @@ -39,7 +40,7 @@ describe('Row formatter', () => { spec: { id, type, version, timeFieldName, fields: JSON.parse(fields), title }, fieldFormats: fieldFormatsMock, shortDotsEnable: false, - metaFields: [], + metaFields: ['_id', '_type', '_score'], }); }; @@ -47,26 +48,15 @@ describe('Row formatter', () => { const fieldsToShow = indexPattern.fields.getAll().map((fld) => fld.name); - // Realistic response with alphabetical insertion order - const formatHitReturnValue = { - also: 'with \\"quotes\\" or 'single qoutes'', - foo: 'bar', - number: '42', - hello: '<h1>World</h1>', - _id: 'a', - _type: 'doc', - _score: 1, - }; - - const formatHitMock = jest.fn().mockReturnValue(formatHitReturnValue); - beforeEach(() => { - // @ts-expect-error - indexPattern.formatHit = formatHitMock; setServices({ uiSettings: { get: () => 100, }, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, } as unknown as DiscoverServices); }); @@ -77,32 +67,32 @@ describe('Row formatter', () => { Array [ Array [ "also", - "with \\\\"quotes\\\\" or 'single qoutes'", + "with \\"quotes\\" or 'single quotes'", ], Array [ "foo", "bar", ], Array [ - "number", - "42", + "hello", + "

World

", ], Array [ - "hello", - "<h1>World</h1>", + "number", + 42, ], Array [ "_id", "a", ], - Array [ - "_type", - "doc", - ], Array [ "_score", 1, ], + Array [ + "_type", + "doc", + ], ] } /> @@ -114,6 +104,10 @@ describe('Row formatter', () => { uiSettings: { get: () => 1, }, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, } as unknown as DiscoverServices); expect(formatRow(hit, indexPattern, [])).toMatchInlineSnapshot(` { Array [ Array [ "also", - "with \\\\"quotes\\\\" or 'single qoutes'", + "with \\"quotes\\" or 'single quotes'", + ], + Array [ + "foo", + "bar", + ], + Array [ + "hello", + "

World

", + ], + Array [ + "number", + 42, + ], + Array [ + "_id", + "a", + ], + Array [ + "_score", + 1, + ], + Array [ + "_type", + "doc", ], ] } @@ -130,18 +148,18 @@ describe('Row formatter', () => { }); it('formats document with highlighted fields first', () => { - expect(formatRow({ ...hit, highlight: { number: '42' } }, indexPattern, fieldsToShow)) + expect(formatRow({ ...hit, highlight: { number: ['42'] } }, indexPattern, fieldsToShow)) .toMatchInlineSnapshot(` { ], Array [ "hello", - "<h1>World</h1>", + "

World

", ], Array [ "_id", "a", ], - Array [ - "_type", - "doc", - ], Array [ "_score", 1, ], + Array [ + "_type", + "doc", + ], ] } /> diff --git a/src/plugins/discover/public/application/apps/main/components/doc_table/lib/row_formatter.tsx b/src/plugins/discover/public/application/apps/main/components/doc_table/lib/row_formatter.tsx index 14cf1839107e70..2702a232f21ef6 100644 --- a/src/plugins/discover/public/application/apps/main/components/doc_table/lib/row_formatter.tsx +++ b/src/plugins/discover/public/application/apps/main/components/doc_table/lib/row_formatter.tsx @@ -6,15 +6,17 @@ * Side Public License, v 1. */ +import { estypes } from '@elastic/elasticsearch'; import React, { Fragment } from 'react'; import type { IndexPattern } from 'src/plugins/data/common'; import { MAX_DOC_FIELDS_DISPLAYED } from '../../../../../../../common'; import { getServices } from '../../../../../../kibana_services'; +import { formatHit } from '../../../../../helpers/format_hit'; import './row_formatter.scss'; interface Props { - defPairs: Array<[string, unknown]>; + defPairs: Array<[string, string]>; } const TemplateComponent = ({ defPairs }: Props) => { return ( @@ -24,8 +26,8 @@ const TemplateComponent = ({ defPairs }: Props) => {
{pair[0]}:
{' '} ))} @@ -34,30 +36,12 @@ const TemplateComponent = ({ defPairs }: Props) => { }; export const formatRow = ( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - hit: Record, + hit: estypes.SearchHit, indexPattern: IndexPattern, fieldsToShow: string[] ) => { - const highlights = hit?.highlight ?? {}; - // Keys are sorted in the hits object - const formatted = indexPattern.formatHit(hit); - const fields = indexPattern.fields; - const highlightPairs: Array<[string, unknown]> = []; - const sourcePairs: Array<[string, unknown]> = []; - Object.entries(formatted).forEach(([key, val]) => { - const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined; - const pairs = highlights[key] ? highlightPairs : sourcePairs; - if (displayKey) { - if (fieldsToShow.includes(displayKey)) { - pairs.push([displayKey, val]); - } - } else { - pairs.push([key, val]); - } - }); - const maxEntries = getServices().uiSettings.get(MAX_DOC_FIELDS_DISPLAYED); - return ; + const pairs = formatHit(hit, indexPattern, fieldsToShow); + return ; }; export const formatTopLevelObject = ( @@ -68,8 +52,8 @@ export const formatTopLevelObject = ( indexPattern: IndexPattern ) => { const highlights = row.highlight ?? {}; - const highlightPairs: Array<[string, unknown]> = []; - const sourcePairs: Array<[string, unknown]> = []; + const highlightPairs: Array<[string, string]> = []; + const sourcePairs: Array<[string, string]> = []; const sorted = Object.entries(fields).sort(([keyA], [keyB]) => keyA.localeCompare(keyB)); sorted.forEach(([key, values]) => { const field = indexPattern.getFieldByName(key); diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_documents.test.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_documents.test.tsx index e5212e877e8ba1..60540268dcd7f3 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_documents.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_documents.test.tsx @@ -20,6 +20,11 @@ import { DiscoverDocuments } from './discover_documents'; import { ElasticSearchHit } from '../../../../doc_views/doc_views_types'; import { indexPatternMock } from '../../../../../__mocks__/index_pattern'; +jest.mock('../../../../../kibana_services', () => ({ + ...jest.requireActual('../../../../../kibana_services'), + getServices: () => jest.requireActual('../../../../../__mocks__/services').discoverServiceMock, +})); + setHeaderActionMenuMounter(jest.fn()); function getProps(fetchStatus: FetchStatus, hits: ElasticSearchHit[]) { diff --git a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.test.tsx b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.test.tsx index 6ebed3185e2f12..7e3252dce1ef58 100644 --- a/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.test.tsx +++ b/src/plugins/discover/public/application/apps/main/components/layout/discover_layout.test.tsx @@ -33,6 +33,19 @@ import { RequestAdapter } from '../../../../../../../inspector'; import { Chart } from '../chart/point_series'; import { DiscoverSidebar } from '../sidebar/discover_sidebar'; +jest.mock('../../../../../kibana_services', () => ({ + ...jest.requireActual('../../../../../kibana_services'), + getServices: () => ({ + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, + uiSettings: { + get: jest.fn((key: string) => key === 'discover:maxDocFieldsDisplayed' && 50), + }, + }), +})); + setHeaderActionMenuMounter(jest.fn()); function getProps(indexPattern: IndexPattern, wasSidebarClosed?: boolean): DiscoverLayoutProps { diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/__snapshots__/discover_index_pattern_management.test.tsx.snap b/src/plugins/discover/public/application/apps/main/components/sidebar/__snapshots__/discover_index_pattern_management.test.tsx.snap index ebb06e0b2ecd36..02e2879476a5e6 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/__snapshots__/discover_index_pattern_management.test.tsx.snap +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/__snapshots__/discover_index_pattern_management.test.tsx.snap @@ -115,42 +115,7 @@ exports[`Discover IndexPattern Management renders correctly 1`] = ` "deserialize": [MockFunction], "getByFieldType": [MockFunction], "getDefaultConfig": [MockFunction], - "getDefaultInstance": [MockFunction] { - "calls": Array [ - Array [ - "string", - ], - Array [ - "string", - ], - Array [ - "string", - ], - ], - "results": Array [ - Object { - "type": "return", - "value": Object { - "convert": [MockFunction], - "getConverterFor": [MockFunction], - }, - }, - Object { - "type": "return", - "value": Object { - "convert": [MockFunction], - "getConverterFor": [MockFunction], - }, - }, - Object { - "type": "return", - "value": Object { - "convert": [MockFunction], - "getConverterFor": [MockFunction], - }, - }, - ], - }, + "getDefaultInstance": [MockFunction], "getDefaultInstanceCacheResolver": [MockFunction], "getDefaultInstancePlain": [MockFunction], "getDefaultType": [MockFunction], @@ -651,8 +616,6 @@ exports[`Discover IndexPattern Management renders correctly 1`] = ` }, ], "flattenHit": [Function], - "formatField": [Function], - "formatHit": [Function], "getFieldAttrs": [Function], "getOriginalSavedObjectBody": [Function], "id": "logstash-*", diff --git a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/field_calculator.js b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/field_calculator.js index be7e9c616273db..c709f3311105d1 100644 --- a/src/plugins/discover/public/application/apps/main/components/sidebar/lib/field_calculator.js +++ b/src/plugins/discover/public/application/apps/main/components/sidebar/lib/field_calculator.js @@ -13,7 +13,7 @@ import { flattenHit } from '../../../../../../../../data/common'; function getFieldValues(hits, field, indexPattern) { const name = field.name; return map(hits, function (hit) { - return flattenHit(hit, indexPattern)[name]; + return flattenHit(hit, indexPattern, { includeIgnoredValues: true })[name]; }); } diff --git a/src/plugins/discover/public/application/apps/main/utils/calc_field_counts.ts b/src/plugins/discover/public/application/apps/main/utils/calc_field_counts.ts index 211c4e5c8b0698..2198d2f66b6b45 100644 --- a/src/plugins/discover/public/application/apps/main/utils/calc_field_counts.ts +++ b/src/plugins/discover/public/application/apps/main/utils/calc_field_counts.ts @@ -22,7 +22,7 @@ export function calcFieldCounts( return {}; } for (const hit of rows) { - const fields = Object.keys(flattenHit(hit, indexPattern)); + const fields = Object.keys(flattenHit(hit, indexPattern, { includeIgnoredValues: true })); for (const fieldName of fields) { counts[fieldName] = (counts[fieldName] || 0) + 1; } diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx index b2be40c0082006..22284480afc055 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx @@ -19,6 +19,11 @@ import { DiscoverServices } from '../../../build_services'; import { ElasticSearchHit } from '../../doc_views/doc_views_types'; import { getDocId } from './discover_grid_document_selection'; +jest.mock('../../../kibana_services', () => ({ + ...jest.requireActual('../../../kibana_services'), + getServices: () => jest.requireActual('../../../__mocks__/services').discoverServiceMock, +})); + function getProps() { const servicesMock = { uiSettings: uiSettingsMock, diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx index 11323080274a98..ca403c813010b8 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx @@ -271,7 +271,11 @@ export const DiscoverGrid = ({ getRenderCellValueFn( indexPattern, displayedRows, - displayedRows ? displayedRows.map((hit) => flattenHit(hit, indexPattern)) : [], + displayedRows + ? displayedRows.map((hit) => + flattenHit(hit, indexPattern, { includeIgnoredValues: true }) + ) + : [], useNewFieldsApi, fieldsToShow, services.uiSettings.get(MAX_DOC_FIELDS_DISPLAYED) diff --git a/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.test.tsx b/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.test.tsx index 6556876217953c..3fb96ba9e9daac 100644 --- a/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.test.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.test.tsx @@ -25,6 +25,9 @@ jest.mock('../../../kibana_services', () => ({ uiSettings: { get: jest.fn(), }, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => (value ? value : '-') })), + }, }), })); @@ -102,7 +105,7 @@ describe('Discover grid cell rendering', function () { rowsSource, rowsSource.map(flatten), false, - [], + ['extension', 'bytes'], 100 ); const component = shallow( @@ -133,7 +136,7 @@ describe('Discover grid cell rendering', function () { } /> - bytes + bytesDisplayName + + _index + + + + _score + + `); }); @@ -196,7 +221,7 @@ describe('Discover grid cell rendering', function () { rowsFields, rowsFields.map(flatten), true, - [], + ['extension', 'bytes'], 100 ); const component = shallow( @@ -229,7 +254,7 @@ describe('Discover grid cell rendering', function () { } /> - bytes + bytesDisplayName + + _index + + + + _score + + `); }); @@ -251,7 +298,7 @@ describe('Discover grid cell rendering', function () { rowsFields, rowsFields.map(flatten), true, - [], + ['extension', 'bytes'], // this is the number of rendered items 1 ); @@ -284,6 +331,41 @@ describe('Discover grid cell rendering', function () { } } /> + + bytesDisplayName + + + + _index + + + + _score + + `); }); @@ -342,7 +424,7 @@ describe('Discover grid cell rendering', function () { rowsFieldsWithTopLevelObject, rowsFieldsWithTopLevelObject.map(flatten), true, - [], + ['object.value', 'extension', 'bytes'], 100 ); const component = shallow( @@ -368,7 +450,7 @@ describe('Discover grid cell rendering', function () { className="dscDiscoverGrid__descriptionListDescription" dangerouslySetInnerHTML={ Object { - "__html": "formatted", + "__html": "100", } } /> @@ -383,7 +465,7 @@ describe('Discover grid cell rendering', function () { rowsFieldsWithTopLevelObject, rowsFieldsWithTopLevelObject.map(flatten), true, - [], + ['extension', 'bytes', 'object.value'], 100 ); const component = shallow( diff --git a/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx b/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx index a0529715806666..4066c13f6391e7 100644 --- a/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx @@ -22,6 +22,8 @@ import { DiscoverGridContext } from './discover_grid_context'; import { JsonCodeEditor } from '../json_code_editor/json_code_editor'; import { defaultMonacoEditorWidth } from './constants'; import { EsHitRecord } from '../../types'; +import { formatFieldValue } from '../../helpers/format_value'; +import { formatHit } from '../../helpers/format_hit'; export const getRenderCellValueFn = ( @@ -145,39 +147,19 @@ export const getRenderCellValueFn = // eslint-disable-next-line @typescript-eslint/no-explicit-any return ; } - const formatted = indexPattern.formatHit(row); - - // Put the most important fields first - const highlights: Record = (row.highlight as Record) ?? {}; - const highlightPairs: Array<[string, string]> = []; - const sourcePairs: Array<[string, string]> = []; - Object.entries(formatted).forEach(([key, val]) => { - const pairs = highlights[key] ? highlightPairs : sourcePairs; - const displayKey = indexPattern.fields.getByName - ? indexPattern.fields.getByName(key)?.displayName - : undefined; - if (displayKey) { - if (fieldsToShow.includes(displayKey)) { - pairs.push([displayKey, val as string]); - } - } else { - pairs.push([key, val as string]); - } - }); + const pairs = formatHit(row, indexPattern, fieldsToShow); return ( - {[...highlightPairs, ...sourcePairs] - .slice(0, maxDocFieldsDisplayed) - .map(([key, value]) => ( - - {key} - - - ))} + {pairs.map(([key, value]) => ( + + {key} + + + ))} ); } @@ -191,12 +173,13 @@ export const getRenderCellValueFn = return {JSON.stringify(rowFlattened[columnId])}; } - const valueFormatted = indexPattern.formatField(row, columnId); - if (typeof valueFormatted === 'undefined') { - return -; - } return ( - // eslint-disable-next-line react/no-danger - + ); }; diff --git a/src/plugins/discover/public/application/components/table/table.test.tsx b/src/plugins/discover/public/application/components/table/table.test.tsx index ce914edcec7030..e61333cce11660 100644 --- a/src/plugins/discover/public/application/components/table/table.test.tsx +++ b/src/plugins/discover/public/application/components/table/table.test.tsx @@ -27,6 +27,10 @@ import { getServices } from '../../../kibana_services'; } }, }, + fieldFormats: { + getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), + }, })); const indexPattern = { @@ -65,8 +69,7 @@ const indexPattern = { ], }, metaFields: ['_index', '_score'], - flattenHit: jest.fn(), - formatHit: jest.fn((hit) => hit._source), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), } as unknown as IndexPattern; indexPattern.fields.getByName = (name: string) => { @@ -359,32 +362,7 @@ describe('DocViewTable at Discover Doc with Fields API', () => { ], }, metaFields: ['_index', '_type', '_score', '_id'], - flattenHit: jest.fn((hit) => { - const result = {} as Record; - Object.keys(hit).forEach((key) => { - if (key !== 'fields') { - result[key] = hit[key]; - } else { - Object.keys(hit.fields).forEach((field) => { - result[field] = hit.fields[field]; - }); - } - }); - return result; - }), - formatHit: jest.fn((hit) => { - const result = {} as Record; - Object.keys(hit).forEach((key) => { - if (key !== 'fields') { - result[key] = hit[key]; - } else { - Object.keys(hit.fields).forEach((field) => { - result[field] = hit.fields[field]; - }); - } - }); - return result; - }), + getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })), } as unknown as IndexPattern; indexPatterneCommerce.fields.getByName = (name: string) => { diff --git a/src/plugins/discover/public/application/components/table/table.tsx b/src/plugins/discover/public/application/components/table/table.tsx index e64dbd10f78553..78a6d9ddd32371 100644 --- a/src/plugins/discover/public/application/components/table/table.tsx +++ b/src/plugins/discover/public/application/components/table/table.tsx @@ -20,6 +20,8 @@ import { } from '../../doc_views/doc_views_types'; import { ACTIONS_COLUMN, MAIN_COLUMNS } from './table_columns'; import { getFieldsToShow } from '../../helpers/get_fields_to_show'; +import { getIgnoredReason, IgnoredReason } from '../../helpers/get_ignored_reason'; +import { formatFieldValue } from '../../helpers/format_value'; export interface DocViewerTableProps { columns?: string[]; @@ -46,6 +48,7 @@ export interface FieldRecord { }; value: { formattedValue: string; + ignored?: IgnoredReason; }; } @@ -64,8 +67,6 @@ export const DocViewerTable = ({ [indexPattern?.fields] ); - const formattedHit = useMemo(() => indexPattern?.formatHit(hit, 'html'), [hit, indexPattern]); - const tableColumns = useMemo(() => { return filter ? [ACTIONS_COLUMN, ...MAIN_COLUMNS] : MAIN_COLUMNS; }, [filter]); @@ -96,7 +97,7 @@ export const DocViewerTable = ({ return null; } - const flattened = flattenHit(hit, indexPattern, { source: true }); + const flattened = flattenHit(hit, indexPattern, { source: true, includeIgnoredValues: true }); const fieldsToShow = getFieldsToShow(Object.keys(flattened), indexPattern, showMultiFields); const items: FieldRecord[] = Object.keys(flattened) @@ -115,6 +116,8 @@ export const DocViewerTable = ({ const displayName = fieldMapping?.displayName ?? field; const fieldType = isNestedFieldParent(field, indexPattern) ? 'nested' : fieldMapping?.type; + const ignored = getIgnoredReason(fieldMapping ?? field, hit._ignored); + return { action: { onToggleColumn, @@ -130,7 +133,8 @@ export const DocViewerTable = ({ scripted: Boolean(fieldMapping?.scripted), }, value: { - formattedValue: formattedHit[field], + formattedValue: formatFieldValue(flattened[field], hit, indexPattern, fieldMapping), + ignored, }, }; }); diff --git a/src/plugins/discover/public/application/components/table/table_cell_actions.tsx b/src/plugins/discover/public/application/components/table/table_cell_actions.tsx index 7f2f87e7c296cb..e43a17448de2ed 100644 --- a/src/plugins/discover/public/application/components/table/table_cell_actions.tsx +++ b/src/plugins/discover/public/application/components/table/table_cell_actions.tsx @@ -21,6 +21,7 @@ interface TableActionsProps { fieldMapping?: IndexPatternField; onFilter: DocViewFilterFn; onToggleColumn: (field: string) => void; + ignoredValue: boolean; } export const TableActions = ({ @@ -30,15 +31,16 @@ export const TableActions = ({ flattenedField, onToggleColumn, onFilter, + ignoredValue, }: TableActionsProps) => { return (
onFilter(fieldMapping, flattenedField, '+')} /> onFilter(fieldMapping, flattenedField, '-')} /> ; +interface IgnoreWarningProps { + reason: IgnoredReason; + rawValue: unknown; +} -export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps) => { +const IgnoreWarning: React.FC = React.memo(({ rawValue, reason }) => { + const multiValue = Array.isArray(rawValue) && rawValue.length > 1; + + const getToolTipContent = (): string => { + switch (reason) { + case IgnoredReason.IGNORE_ABOVE: + return multiValue + ? i18n.translate('discover.docView.table.ignored.multiAboveTooltip', { + defaultMessage: `One or more values in this field are too long and can't be searched or filtered.`, + }) + : i18n.translate('discover.docView.table.ignored.singleAboveTooltip', { + defaultMessage: `The value in this field is too long and can't be searched or filtered.`, + }); + case IgnoredReason.MALFORMED: + return multiValue + ? i18n.translate('discover.docView.table.ignored.multiMalformedTooltip', { + defaultMessage: `This field has one or more malformed values that can't be searched or filtered.`, + }) + : i18n.translate('discover.docView.table.ignored.singleMalformedTooltip', { + defaultMessage: `The value in this field is malformed and can't be searched or filtered.`, + }); + case IgnoredReason.UNKNOWN: + return multiValue + ? i18n.translate('discover.docView.table.ignored.multiUnknownTooltip', { + defaultMessage: `One or more values in this field were ignored by Elasticsearch and can't be searched or filtered.`, + }) + : i18n.translate('discover.docView.table.ignored.singleUnknownTooltip', { + defaultMessage: `The value in this field was ignored by Elasticsearch and can't be searched or filtered.`, + }); + } + }; + + return ( + + + + + + + + {multiValue + ? i18n.translate('discover.docViews.table.ignored.multiValueLabel', { + defaultMessage: 'Contains ignored values', + }) + : i18n.translate('discover.docViews.table.ignored.singleValueLabel', { + defaultMessage: 'Ignored value', + })} + + + + + ); +}); + +type TableFieldValueProps = Pick & { + formattedValue: FieldRecord['value']['formattedValue']; + rawValue: unknown; + ignoreReason?: IgnoredReason; +}; + +export const TableFieldValue = ({ + formattedValue, + field, + rawValue, + ignoreReason, +}: TableFieldValueProps) => { const [fieldOpen, setFieldOpen] = useState(false); - const value = String(formattedValue); + const value = String(rawValue); const isCollapsible = value.length > COLLAPSE_LINE_LENGTH; const isCollapsed = isCollapsible && !fieldOpen; @@ -32,18 +111,26 @@ export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps) return ( - {isCollapsible && ( - + {(isCollapsible || ignoreReason) && ( + + {isCollapsible && ( + + + + )} + {ignoreReason && ( + + + + )} + )}
); diff --git a/src/plugins/discover/public/application/components/table/table_columns.tsx b/src/plugins/discover/public/application/components/table/table_columns.tsx index 5bd92fe9166e9c..5944f9bede6466 100644 --- a/src/plugins/discover/public/application/components/table/table_columns.tsx +++ b/src/plugins/discover/public/application/components/table/table_columns.tsx @@ -31,7 +31,7 @@ export const ACTIONS_COLUMN: EuiBasicTableColumn = { ), render: ( { flattenedField, isActive, onFilter, onToggleColumn }: FieldRecord['action'], - { field: { field, fieldMapping } }: FieldRecord + { field: { field, fieldMapping }, value: { ignored } }: FieldRecord ) => { return ( = { flattenedField={flattenedField} onFilter={onFilter!} onToggleColumn={onToggleColumn} + ignoredValue={!!ignored} /> ); }, @@ -82,8 +83,18 @@ export const MAIN_COLUMNS: Array> = [ ), - render: ({ formattedValue }: FieldRecord['value'], { field: { field } }: FieldRecord) => { - return ; + render: ( + { formattedValue, ignored }: FieldRecord['value'], + { field: { field }, action: { flattenedField } }: FieldRecord + ) => { + return ( + + ); }, }, ]; diff --git a/src/plugins/discover/public/application/components/table/table_row_btn_filter_add.tsx b/src/plugins/discover/public/application/components/table/table_row_btn_filter_add.tsx index 5fe1b4dc33342a..de56d733442d6f 100644 --- a/src/plugins/discover/public/application/components/table/table_row_btn_filter_add.tsx +++ b/src/plugins/discover/public/application/components/table/table_row_btn_filter_add.tsx @@ -20,7 +20,7 @@ export function DocViewTableRowBtnFilterAdd({ onClick, disabled = false }: Props const tooltipContent = disabled ? ( ) : ( ) : ( ({ + getServices: () => jest.requireActual('../../__mocks__/services').discoverServiceMock, +})); + +describe('formatHit', () => { + let hit: estypes.SearchHit; + beforeEach(() => { + hit = { + _id: '1', + _index: 'logs', + fields: { + message: ['foobar'], + extension: ['png'], + 'object.value': [42, 13], + bytes: [123], + }, + }; + (dataViewMock.getFormatterForField as jest.Mock).mockReturnValue({ + convert: (value: unknown) => `formatted:${value}`, + }); + }); + + afterEach(() => { + (discoverServiceMock.uiSettings.get as jest.Mock).mockReset(); + }); + + it('formats a document as expected', () => { + const formatted = formatHit(hit, dataViewMock, ['message', 'extension', 'object.value']); + expect(formatted).toEqual([ + ['extension', 'formatted:png'], + ['message', 'formatted:foobar'], + ['object.value', 'formatted:42,13'], + ['_index', 'formatted:logs'], + ['_score', undefined], + ]); + }); + + it('orders highlighted fields first', () => { + const formatted = formatHit({ ...hit, highlight: { message: ['%%'] } }, dataViewMock, [ + 'message', + 'extension', + 'object.value', + ]); + expect(formatted.map(([fieldName]) => fieldName)).toEqual([ + 'message', + 'extension', + 'object.value', + '_index', + '_score', + ]); + }); + + it('only limits count of pairs based on advanced setting', () => { + (discoverServiceMock.uiSettings.get as jest.Mock).mockImplementation( + (key) => key === MAX_DOC_FIELDS_DISPLAYED && 2 + ); + const formatted = formatHit(hit, dataViewMock, ['message', 'extension', 'object.value']); + expect(formatted).toEqual([ + ['extension', 'formatted:png'], + ['message', 'formatted:foobar'], + ]); + }); + + it('should not include fields not mentioned in fieldsToShow', () => { + const formatted = formatHit(hit, dataViewMock, ['message', 'object.value']); + expect(formatted).toEqual([ + ['message', 'formatted:foobar'], + ['object.value', 'formatted:42,13'], + ['_index', 'formatted:logs'], + ['_score', undefined], + ]); + }); + + it('should filter fields based on their real name not displayName', () => { + const formatted = formatHit(hit, dataViewMock, ['bytes']); + expect(formatted).toEqual([ + ['bytesDisplayName', 'formatted:123'], + ['_index', 'formatted:logs'], + ['_score', undefined], + ]); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/format_hit.ts b/src/plugins/discover/public/application/helpers/format_hit.ts new file mode 100644 index 00000000000000..3890973a3f3e48 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/format_hit.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { DataView, flattenHit } from '../../../../data/common'; +import { MAX_DOC_FIELDS_DISPLAYED } from '../../../common'; +import { getServices } from '../../kibana_services'; +import { formatFieldValue } from './format_value'; + +const formattedHitCache = new WeakMap(); + +type FormattedHit = Array<[fieldName: string, formattedValue: string]>; + +/** + * Returns a formatted document in form of key/value pairs of the fields name and a formatted value. + * The value returned in each pair is an HTML string which is safe to be applied to the DOM, since + * it's formatted using field formatters. + * @param hit The hit to format + * @param dataView The corresponding data view + * @param fieldsToShow A list of fields that should be included in the document summary. + */ +export function formatHit( + hit: estypes.SearchHit, + dataView: DataView, + fieldsToShow: string[] +): FormattedHit { + const cached = formattedHitCache.get(hit); + if (cached) { + return cached; + } + + const highlights = hit.highlight ?? {}; + // Flatten the object using the flattenHit implementation we use across Discover for flattening documents. + const flattened = flattenHit(hit, dataView, { includeIgnoredValues: true, source: true }); + + const highlightPairs: Array<[fieldName: string, formattedValue: string]> = []; + const sourcePairs: Array<[fieldName: string, formattedValue: string]> = []; + + // Add each flattened field into the corresponding array for highlighted or other fields, + // depending on whether the original hit had a highlight for it. That way we can later + // put highlighted fields first in the document summary. + Object.entries(flattened).forEach(([key, val]) => { + // Retrieve the (display) name of the fields, if it's a mapped field on the data view + const displayKey = dataView.fields.getByName(key)?.displayName; + const pairs = highlights[key] ? highlightPairs : sourcePairs; + // Format the raw value using the regular field formatters for that field + const formattedValue = formatFieldValue(val, hit, dataView, dataView.fields.getByName(key)); + // If the field was a mapped field, we validate it against the fieldsToShow list, if not + // we always include it into the result. + if (displayKey) { + if (fieldsToShow.includes(key)) { + pairs.push([displayKey, formattedValue]); + } + } else { + pairs.push([key, formattedValue]); + } + }); + const maxEntries = getServices().uiSettings.get(MAX_DOC_FIELDS_DISPLAYED); + const formatted = [...highlightPairs, ...sourcePairs].slice(0, maxEntries); + formattedHitCache.set(hit, formatted); + return formatted; +} diff --git a/src/plugins/discover/public/application/helpers/format_value.test.ts b/src/plugins/discover/public/application/helpers/format_value.test.ts new file mode 100644 index 00000000000000..76d95c08e4a19d --- /dev/null +++ b/src/plugins/discover/public/application/helpers/format_value.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { FieldFormat } from '../../../../field_formats/common'; +import { indexPatternMock } from '../../__mocks__/index_pattern'; +import { formatFieldValue } from './format_value'; + +import { getServices } from '../../kibana_services'; + +jest.mock('../../kibana_services', () => { + const services = { + fieldFormats: { + getDefaultInstance: jest.fn( + () => ({ convert: (value: unknown) => value } as FieldFormat) + ), + }, + }; + return { getServices: () => services }; +}); + +const hit = { + _id: '1', + _index: 'index', + fields: { + message: 'foo', + }, +}; + +describe('formatFieldValue', () => { + afterEach(() => { + (indexPatternMock.getFormatterForField as jest.Mock).mockReset(); + (getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReset(); + }); + + it('should call correct fieldFormatter for field', () => { + const formatterForFieldMock = indexPatternMock.getFormatterForField as jest.Mock; + const convertMock = jest.fn((value: unknown) => `formatted:${value}`); + formatterForFieldMock.mockReturnValue({ convert: convertMock }); + const field = indexPatternMock.fields.getByName('message'); + expect(formatFieldValue('foo', hit, indexPatternMock, field)).toBe('formatted:foo'); + expect(indexPatternMock.getFormatterForField).toHaveBeenCalledWith(field); + expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field, hit }); + }); + + it('should call default string formatter if no field specified', () => { + const convertMock = jest.fn((value: unknown) => `formatted:${value}`); + (getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReturnValue({ + convert: convertMock, + }); + expect(formatFieldValue('foo', hit, indexPatternMock)).toBe('formatted:foo'); + expect(getServices().fieldFormats.getDefaultInstance).toHaveBeenCalledWith('string'); + expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field: undefined, hit }); + }); + + it('should call default string formatter if no indexPattern is specified', () => { + const convertMock = jest.fn((value: unknown) => `formatted:${value}`); + (getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReturnValue({ + convert: convertMock, + }); + expect(formatFieldValue('foo', hit)).toBe('formatted:foo'); + expect(getServices().fieldFormats.getDefaultInstance).toHaveBeenCalledWith('string'); + expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field: undefined, hit }); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/format_value.ts b/src/plugins/discover/public/application/helpers/format_value.ts new file mode 100644 index 00000000000000..cc33276790372e --- /dev/null +++ b/src/plugins/discover/public/application/helpers/format_value.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { DataView, DataViewField, KBN_FIELD_TYPES } from '../../../../data/common'; +import { getServices } from '../../kibana_services'; + +/** + * Formats the value of a specific field using the appropriate field formatter if available + * or the default string field formatter otherwise. + * + * @param value The value to format + * @param hit The actual search hit (required to get highlight information from) + * @param dataView The data view if available + * @param field The field that value was from if available + * @returns An sanitized HTML string, that is safe to be applied via dangerouslySetInnerHTML + */ +export function formatFieldValue( + value: unknown, + hit: estypes.SearchHit, + dataView?: DataView, + field?: DataViewField +): string { + if (!dataView || !field) { + // If either no field is available or no data view, we'll use the default + // string formatter to format that field. + return getServices() + .fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING) + .convert(value, 'html', { hit, field }); + } + + // If we have a data view and field we use that fields field formatter + return dataView.getFormatterForField(field).convert(value, 'html', { hit, field }); +} diff --git a/src/plugins/discover/public/application/helpers/get_ignored_reason.test.ts b/src/plugins/discover/public/application/helpers/get_ignored_reason.test.ts new file mode 100644 index 00000000000000..13632ca5ed901d --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_ignored_reason.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getIgnoredReason, IgnoredReason } from './get_ignored_reason'; +import { DataViewField, KBN_FIELD_TYPES } from '../../../../data/common'; + +function field(params: Partial): DataViewField { + return { + name: 'text', + type: 'keyword', + ...params, + } as unknown as DataViewField; +} + +describe('getIgnoredReason', () => { + it('will correctly return undefined when no value was ignored', () => { + expect(getIgnoredReason(field({ name: 'foo' }), undefined)).toBeUndefined(); + expect(getIgnoredReason(field({ name: 'foo' }), ['bar', 'baz'])).toBeUndefined(); + }); + + it('will return UNKNOWN if the field passed in was only a name, and thus no type information is present', () => { + expect(getIgnoredReason('foo', ['foo'])).toBe(IgnoredReason.UNKNOWN); + }); + + it('will return IGNORE_ABOVE for string types', () => { + expect(getIgnoredReason(field({ name: 'foo', type: KBN_FIELD_TYPES.STRING }), ['foo'])).toBe( + IgnoredReason.IGNORE_ABOVE + ); + }); + + // Each type that can have malformed values + [ + KBN_FIELD_TYPES.DATE, + KBN_FIELD_TYPES.IP, + KBN_FIELD_TYPES.GEO_POINT, + KBN_FIELD_TYPES.GEO_SHAPE, + KBN_FIELD_TYPES.NUMBER, + ].forEach((type) => { + it(`will return MALFORMED for ${type} fields`, () => { + expect(getIgnoredReason(field({ name: 'foo', type }), ['foo'])).toBe(IgnoredReason.MALFORMED); + }); + }); + + it('will return unknown reasons if it does not know what the reason was', () => { + expect(getIgnoredReason(field({ name: 'foo', type: 'range' }), ['foo'])).toBe( + IgnoredReason.UNKNOWN + ); + }); +}); diff --git a/src/plugins/discover/public/application/helpers/get_ignored_reason.ts b/src/plugins/discover/public/application/helpers/get_ignored_reason.ts new file mode 100644 index 00000000000000..4d2fb85bdb2c45 --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_ignored_reason.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { DataViewField, KBN_FIELD_TYPES } from '../../../../data/common'; + +export enum IgnoredReason { + IGNORE_ABOVE = 'ignore_above', + MALFORMED = 'malformed', + UNKNOWN = 'unknown', +} + +/** + * Returns the reason why a specific field was ignored in the response. + * Will return undefined if the field had no ignored values in it. + * This implementation will make some assumptions based on specific types + * of ignored values can only happen with specific field types in Elasticsearch. + * + * @param field Either the data view field or the string name of it. + * @param ignoredFields The hit._ignored value of the hit to validate. + */ +export function getIgnoredReason( + field: DataViewField | string, + ignoredFields: estypes.SearchHit['_ignored'] +): IgnoredReason | undefined { + const fieldName = typeof field === 'string' ? field : field.name; + if (!ignoredFields?.includes(fieldName)) { + return undefined; + } + + if (typeof field === 'string') { + return IgnoredReason.UNKNOWN; + } + + switch (field.type) { + case KBN_FIELD_TYPES.STRING: + return IgnoredReason.IGNORE_ABOVE; + case KBN_FIELD_TYPES.NUMBER: + case KBN_FIELD_TYPES.DATE: + case KBN_FIELD_TYPES.GEO_POINT: + case KBN_FIELD_TYPES.GEO_SHAPE: + case KBN_FIELD_TYPES.IP: + return IgnoredReason.MALFORMED; + default: + return IgnoredReason.UNKNOWN; + } +} diff --git a/src/plugins/discover/public/build_services.ts b/src/plugins/discover/public/build_services.ts index ab2484abee8923..ac16b6b3cc2bab 100644 --- a/src/plugins/discover/public/build_services.ts +++ b/src/plugins/discover/public/build_services.ts @@ -36,6 +36,7 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public'; import { UrlForwardingStart } from '../../url_forwarding/public'; import { NavigationPublicPluginStart } from '../../navigation/public'; import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public'; +import { FieldFormatsStart } from '../../field_formats/public'; import type { SpacesApi } from '../../../../x-pack/plugins/spaces/public'; @@ -49,6 +50,7 @@ export interface DiscoverServices { history: () => History; theme: ChartsPluginStart['theme']; filterManager: FilterManager; + fieldFormats: FieldFormatsStart; indexPatterns: IndexPatternsContract; inspector: InspectorPublicPluginStart; metadata: { branch: string }; @@ -82,6 +84,7 @@ export function buildServices( data: plugins.data, docLinks: core.docLinks, theme: plugins.charts.theme, + fieldFormats: plugins.fieldFormats, filterManager: plugins.data.query.filterManager, history: getHistory, indexPatterns: plugins.data.indexPatterns, diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 39b73d42137213..eee4b8f599b941 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -62,6 +62,7 @@ import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_fie import { DeferredSpinner } from './shared'; import { ViewSavedSearchAction } from './application/embeddable/view_saved_search_action'; import type { SpacesPluginStart } from '../../../../x-pack/plugins/spaces/public'; +import { FieldFormatsStart } from '../../field_formats/public'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -185,6 +186,7 @@ export interface DiscoverStartPlugins { navigation: NavigationStart; charts: ChartsPluginStart; data: DataPublicPluginStart; + fieldFormats: FieldFormatsStart; share?: SharePluginStart; kibanaLegacy: KibanaLegacyStart; urlForwarding: UrlForwardingStart; diff --git a/src/plugins/field_formats/common/converters/source.test.ts b/src/plugins/field_formats/common/converters/source.test.ts index 298c93dac8c4ee..6f9e96a136d0bc 100644 --- a/src/plugins/field_formats/common/converters/source.test.ts +++ b/src/plugins/field_formats/common/converters/source.test.ts @@ -19,7 +19,7 @@ describe('Source Format', () => { convertHtml = source.getConverterFor(HTML_CONTEXT_TYPE) as HtmlContextTypeConvert; }); - test('should use the text content type if a field is not passed', () => { + test('should render stringified object', () => { const hit = { foo: 'bar', number: 42, @@ -27,23 +27,8 @@ describe('Source Format', () => { also: 'with "quotes" or \'single quotes\'', }; - expect(convertHtml(hit)).toBe( - '{"foo":"bar","number":42,"hello":"<h1>World</h1>","also":"with \\"quotes\\" or 'single quotes'"}' - ); - }); - - test('should render a description list if a field is passed', () => { - const hit = { - foo: 'bar', - number: 42, - hello: '

World

', - also: 'with "quotes" or \'single quotes\'', - }; - - expect( - convertHtml(hit, { field: 'field', indexPattern: { formatHit: (h: string) => h }, hit }) - ).toMatchInlineSnapshot( - `"
foo:
bar
number:
42
hello:

World

also:
with \\"quotes\\" or 'single quotes'
"` + expect(convertHtml(hit, { field: 'field', hit })).toMatchInlineSnapshot( + `"{"foo":"bar","number":42,"hello":"<h1>World</h1>","also":"with \\\\"quotes\\\\" or 'single quotes'"}"` ); }); }); diff --git a/src/plugins/field_formats/common/converters/source.tsx b/src/plugins/field_formats/common/converters/source.tsx index 1caffb5bfb9a86..f92027ec07451a 100644 --- a/src/plugins/field_formats/common/converters/source.tsx +++ b/src/plugins/field_formats/common/converters/source.tsx @@ -7,33 +7,8 @@ */ import { KBN_FIELD_TYPES } from '@kbn/field-types'; -import React, { Fragment } from 'react'; -import ReactDOM from 'react-dom/server'; -import { escape, keys } from 'lodash'; -import { shortenDottedString } from '../utils'; import { FieldFormat } from '../field_format'; -import { TextContextTypeConvert, HtmlContextTypeConvert, FIELD_FORMAT_IDS } from '../types'; -import { FORMATS_UI_SETTINGS } from '../constants/ui_settings'; - -interface Props { - defPairs: Array<[string, string]>; -} -const TemplateComponent = ({ defPairs }: Props) => { - return ( -
- {defPairs.map((pair, idx) => ( - -
-
{' '} - - ))} -
- ); -}; +import { TextContextTypeConvert, FIELD_FORMAT_IDS } from '../types'; /** @public */ export class SourceFormat extends FieldFormat { @@ -42,32 +17,4 @@ export class SourceFormat extends FieldFormat { static fieldType = KBN_FIELD_TYPES._SOURCE; textConvert: TextContextTypeConvert = (value: string) => JSON.stringify(value); - - htmlConvert: HtmlContextTypeConvert = (value: string, options = {}) => { - const { field, hit, indexPattern } = options; - - if (!field) { - const converter = this.getConverterFor('text') as Function; - - return escape(converter(value)); - } - - const highlights: Record = (hit && hit.highlight) || {}; - // TODO: remove index pattern dependency - const formatted = hit ? indexPattern!.formatHit(hit) : {}; - const highlightPairs: Array<[string, string]> = []; - const sourcePairs: Array<[string, string]> = []; - const isShortDots = this.getConfig!(FORMATS_UI_SETTINGS.SHORT_DOTS_ENABLE); - - keys(formatted).forEach((key) => { - const pairs = highlights[key] ? highlightPairs : sourcePairs; - const newField = isShortDots ? shortenDottedString(key) : key; - const val = formatted![key]; - pairs.push([newField as string, val]); - }, []); - - return ReactDOM.renderToStaticMarkup( - - ); - }; } diff --git a/src/plugins/field_formats/common/types.ts b/src/plugins/field_formats/common/types.ts index 00f9f5d707e89d..6f0efebe389a10 100644 --- a/src/plugins/field_formats/common/types.ts +++ b/src/plugins/field_formats/common/types.ts @@ -17,10 +17,6 @@ export type FieldFormatsContentType = 'html' | 'text'; */ export interface HtmlContextTypeOptions { field?: { name: string }; - // TODO: get rid of indexPattern dep completely - indexPattern?: { - formatHit: (hit: { highlight: Record }) => Record; - }; hit?: { highlight: Record }; } diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 6c2989d54309d8..77ad4fba1ab607 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -356,7 +356,7 @@ export class CsvGenerator { let table: Datatable | undefined; try { - table = tabifyDocs(results, index, { shallow: true }); + table = tabifyDocs(results, index, { shallow: true, includeIgnoredValues: true }); } catch (err) { this.logger.error(err); } diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f0fd864e190f43..3685e6c0b8cfe3 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -2428,7 +2428,6 @@ "discover.docViews.table.toggleFieldDetails": "フィールド詳細を切り替える", "discover.docViews.table.unableToFilterForPresenceOfMetaFieldsTooltip": "メタフィールドの有無でフィルタリングできません", "discover.docViews.table.unableToFilterForPresenceOfScriptedFieldsTooltip": "スクリプトフィールドの有無でフィルタリングできません", - "discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip": "インデックスされていないフィールドは検索できません", "discover.embeddable.inspectorRequestDataTitle": "データ", "discover.embeddable.inspectorRequestDescription": "このリクエストはElasticsearchにクエリをかけ、検索データを取得します。", "discover.embeddable.search.displayName": "検索", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index d4d327499ddf27..3af347375946a9 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -2453,7 +2453,6 @@ "discover.docViews.table.toggleFieldDetails": "切换字段详细信息", "discover.docViews.table.unableToFilterForPresenceOfMetaFieldsTooltip": "无法筛选元数据字段是否存在", "discover.docViews.table.unableToFilterForPresenceOfScriptedFieldsTooltip": "无法筛选脚本字段是否存在", - "discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip": "无法搜索未索引字段", "discover.embeddable.inspectorRequestDataTitle": "数据", "discover.embeddable.inspectorRequestDescription": "此请求将查询 Elasticsearch 以获取搜索的数据。", "discover.embeddable.search.displayName": "搜索",