diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts index 9c92af30097a21..0fa1ddeee820d9 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts @@ -19,7 +19,7 @@ export interface DataStreamsTabTestBed extends TestBed { goToDataStreamsList: () => void; clickEmptyPromptIndexTemplateLink: () => void; clickIncludeStatsSwitch: () => void; - clickIncludeManagedSwitch: () => void; + toggleViewFilterAt: (index: number) => void; clickReloadButton: () => void; clickNameAt: (index: number) => void; clickIndicesAt: (index: number) => void; @@ -82,9 +82,16 @@ export const setup = async (overridingDependencies: any = {}): Promise { - const { find } = testBed; - find('includeManagedSwitch').simulate('click'); + const toggleViewFilterAt = (index: number) => { + const { find, component } = testBed; + act(() => { + find('viewButton').simulate('click'); + }); + component.update(); + act(() => { + find('filterItem').at(index).simulate('click'); + }); + component.update(); }; const clickReloadButton = () => { @@ -197,7 +204,7 @@ export const setup = async (overridingDependencies: any = {}): Promise): DataSt privileges: { delete_index: true, }, + hidden: false, ...dataStream, }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts index 91502621d50c5e..93899dece33082 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts @@ -19,6 +19,8 @@ import { createNonDataStreamIndex, } from './data_streams_tab.helpers'; +const nonBreakingSpace = ' '; + describe('Data Streams tab', () => { const { server, httpRequestsMockHelpers } = setupEnvironment(); let testBed: DataStreamsTabTestBed; @@ -82,6 +84,25 @@ describe('Data Streams tab', () => { // Assert against the text because the href won't be available, due to dependency upon our core mock. expect(findEmptyPromptIndexTemplateLink().text()).toBe('Fleet'); }); + + test('when hidden data streams are filtered by default, the table is rendered empty', async () => { + const hiddenDataStream = createDataStreamPayload({ + name: 'hidden-data-stream', + hidden: true, + }); + httpRequestsMockHelpers.setLoadDataStreamsResponse([hiddenDataStream]); + + testBed = await setup({ + plugins: {}, + }); + + await act(async () => { + testBed.actions.goToDataStreamsList(); + }); + + testBed.component.update(); + expect(testBed.find('dataStreamTable').text()).toContain('No data streams found'); + }); }); describe('when there are data streams', () => { @@ -397,7 +418,6 @@ describe('Data Streams tab', () => { }); describe('managed data streams', () => { - const nonBreakingSpace = ' '; beforeEach(async () => { const managedDataStream = createDataStreamPayload({ name: 'managed-data-stream', @@ -429,8 +449,8 @@ describe('Data Streams tab', () => { ]); }); - test('turning off "Include managed" switch hides managed data streams', async () => { - const { exists, actions, component, table } = testBed; + test('turning off "managed" filter hides managed data streams', async () => { + const { actions, table } = testBed; let { tableCellsValues } = table.getMetaData('dataStreamTable'); expect(tableCellsValues).toEqual([ @@ -438,15 +458,40 @@ describe('Data Streams tab', () => { ['', 'non-managed-data-stream', 'green', '1', 'Delete'], ]); - expect(exists('includeManagedSwitch')).toBe(true); + actions.toggleViewFilterAt(0); + + ({ tableCellsValues } = table.getMetaData('dataStreamTable')); + expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]); + }); + }); + + describe('hidden data streams', () => { + beforeEach(async () => { + const hiddenDataStream = createDataStreamPayload({ + name: 'hidden-data-stream', + hidden: true, + }); + httpRequestsMockHelpers.setLoadDataStreamsResponse([hiddenDataStream]); + testBed = await setup({ + history: createMemoryHistory(), + }); await act(async () => { - actions.clickIncludeManagedSwitch(); + testBed.actions.goToDataStreamsList(); }); - component.update(); + testBed.component.update(); + }); - ({ tableCellsValues } = table.getMetaData('dataStreamTable')); - expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]); + test('show hidden data streams when filter is toggled', () => { + const { table, actions } = testBed; + + actions.toggleViewFilterAt(1); + + const { tableCellsValues } = table.getMetaData('dataStreamTable'); + + expect(tableCellsValues).toEqual([ + ['', `hidden-data-stream${nonBreakingSpace}Hidden`, 'green', '1', 'Delete'], + ]); }); }); diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts index fe7db99c98db1f..333cb4b97f2aac 100644 --- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts @@ -19,6 +19,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS maximum_timestamp: maxTimeStamp, _meta, privileges, + hidden, } = dataStreamFromEs; return { @@ -39,6 +40,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS maxTimeStamp, _meta, privileges, + hidden, }; } diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts index fdfe6278eb9859..fca10f85ab63c2 100644 --- a/x-pack/plugins/index_management/common/types/data_streams.ts +++ b/x-pack/plugins/index_management/common/types/data_streams.ts @@ -38,6 +38,7 @@ export interface DataStreamFromEs { store_size?: string; maximum_timestamp?: number; privileges: PrivilegesFromEs; + hidden: boolean; } export interface DataStreamIndexFromEs { @@ -59,6 +60,7 @@ export interface DataStream { maxTimeStamp?: number; _meta?: Meta; privileges: Privileges; + hidden: boolean; } export interface DataStreamIndex { diff --git a/x-pack/plugins/index_management/public/application/lib/data_streams.tsx b/x-pack/plugins/index_management/public/application/lib/data_streams.tsx index ca5297e3993391..93791f8a582245 100644 --- a/x-pack/plugins/index_management/public/application/lib/data_streams.tsx +++ b/x-pack/plugins/index_management/public/application/lib/data_streams.tsx @@ -6,10 +6,34 @@ import { DataStream } from '../../../common'; -export const isManagedByIngestManager = (dataStream: DataStream): boolean => { +export const isFleetManaged = (dataStream: DataStream): boolean => { + // TODO check if the wording will change to 'fleet' return Boolean(dataStream._meta?.managed && dataStream._meta?.managed_by === 'ingest-manager'); }; -export const filterDataStreams = (dataStreams: DataStream[]): DataStream[] => { - return dataStreams.filter((dataStream: DataStream) => !isManagedByIngestManager(dataStream)); +export const filterDataStreams = ( + dataStreams: DataStream[], + visibleTypes: string[] +): DataStream[] => { + return dataStreams.filter((dataStream: DataStream) => { + // include all data streams that are neither hidden nor managed + if (!dataStream.hidden && !isFleetManaged(dataStream)) { + return true; + } + if (dataStream.hidden && visibleTypes.includes('hidden')) { + return true; + } + return isFleetManaged(dataStream) && visibleTypes.includes('managed'); + }); +}; + +export const isSelectedDataStreamHidden = ( + dataStreams: DataStream[], + selectedDataStreamName?: string +): boolean => { + return ( + !!selectedDataStreamName && + !!dataStreams.find((dataStream: DataStream) => dataStream.name === selectedDataStreamName) + ?.hidden + ); }; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/components/filter_list_button.tsx b/x-pack/plugins/index_management/public/application/sections/home/components/filter_list_button.tsx similarity index 100% rename from x-pack/plugins/index_management/public/application/sections/home/template_list/components/filter_list_button.tsx rename to x-pack/plugins/index_management/public/application/sections/home/components/filter_list_button.tsx diff --git a/x-pack/plugins/index_management/public/application/sections/home/components/index.ts b/x-pack/plugins/index_management/public/application/sections/home/components/index.ts new file mode 100644 index 00000000000000..3df506583b65a7 --- /dev/null +++ b/x-pack/plugins/index_management/public/application/sections/home/components/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { FilterListButton, Filters } from './filter_list_button'; diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_badges.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_badges.tsx new file mode 100644 index 00000000000000..e86dfe7f28585a --- /dev/null +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_badges.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiBadge, EuiBadgeGroup } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { DataStream } from '../../../../../common'; +import { isFleetManaged } from '../../../lib/data_streams'; + +interface Props { + dataStream: DataStream; +} + +export const DataStreamsBadges: React.FunctionComponent = ({ dataStream }) => { + const badges = []; + if (isFleetManaged(dataStream)) { + badges.push( + + + + ); + } + if (dataStream.hidden) { + badges.push( + + + + ); + } + return badges.length > 0 ? ( + <> +   + {badges} + + ) : null; +}; diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx index ec47b2c062aa99..33fd1b3f18716c 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx @@ -32,6 +32,7 @@ import { useUrlGenerator } from '../../../../services/use_url_generator'; import { getIndexListUri, getTemplateDetailsLink } from '../../../../services/routing'; import { ILM_PAGES_POLICY_EDIT, ILM_URL_GENERATOR_ID } from '../../../../constants'; import { useAppContext } from '../../../../app_context'; +import { DataStreamsBadges } from '../data_stream_badges'; interface DetailsListProps { details: Array<{ @@ -269,6 +270,7 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({

{dataStreamName} + {dataStream && }

diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx index f43b9799082a0c..64d874c76afb3c 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { RouteComponentProps } from 'react-router-dom'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -32,8 +32,10 @@ import { documentationService } from '../../../services/documentation'; import { Section } from '../home'; import { DataStreamTable } from './data_stream_table'; import { DataStreamDetailPanel } from './data_stream_detail_panel'; -import { filterDataStreams } from '../../../lib/data_streams'; +import { filterDataStreams, isSelectedDataStreamHidden } from '../../../lib/data_streams'; +import { FilterListButton, Filters } from '../components'; +export type DataStreamFilterName = 'managed' | 'hidden'; interface MatchParams { dataStreamName?: string; } @@ -45,7 +47,7 @@ export const DataStreamList: React.FunctionComponent { - const { isDeepLink } = extractQueryParams(search); + const { isDeepLink, includeHidden } = extractQueryParams(search); const decodedDataStreamName = attemptToURIDecode(dataStreamName); const { @@ -54,11 +56,111 @@ export const DataStreamList: React.FunctionComponent>({ + managed: { + name: i18n.translate('xpack.idxMgmt.dataStreamList.viewManagedLabel', { + defaultMessage: 'Fleet-managed data streams', + }), + checked: 'on', + }, + hidden: { + name: i18n.translate('xpack.idxMgmt.dataStreamList.viewHiddenLabel', { + defaultMessage: 'Hidden data streams', + }), + checked: includeHidden ? 'on' : 'off', + }, + }); + + const activateHiddenFilter = (shouldBeActive: boolean) => { + if (shouldBeActive && filters.hidden.checked === 'off') { + setFilters({ + ...filters, + hidden: { + ...filters.hidden, + checked: 'on', + }, + }); + } + }; + + const filteredDataStreams = useMemo(() => { + if (!dataStreams) { + // If dataStreams are not fetched, return empty array. + return []; + } + + const visibleTypes = Object.entries(filters) + .filter(([name, _filter]) => _filter.checked === 'on') + .map(([name]) => name); + + return filterDataStreams(dataStreams, visibleTypes); + }, [dataStreams, filters]); + + const renderHeader = () => { + return ( + + + + + {i18n.translate('xpack.idxMgmt.dataStreamListDescription.learnMoreLinkText', { + defaultMessage: 'Learn more.', + })} + + ), + }} + /> + + + + + + + setIsIncludeStatsChecked(e.target.checked)} + data-test-subj="includeStatsSwitch" + /> + + + + + + + + + filters={filters} onChange={setFilters} /> + + + ); + }; + let content; if (isLoading) { @@ -150,94 +252,10 @@ export const DataStreamList: React.FunctionComponent ); } else if (Array.isArray(dataStreams) && dataStreams.length > 0) { - const filteredDataStreams = isIncludeManagedChecked - ? dataStreams - : filterDataStreams(dataStreams); + activateHiddenFilter(isSelectedDataStreamHidden(dataStreams, decodedDataStreamName)); content = ( <> - - - - - {i18n.translate('xpack.idxMgmt.dataStreamListDescription.learnMoreLinkText', { - defaultMessage: 'Learn more.', - })} - - ), - }} - /> - - - - - - - setIsIncludeStatsChecked(e.target.checked)} - data-test-subj="includeStatsSwitch" - /> - - - - - - - - - - - setIsIncludeManagedChecked(e.target.checked)} - data-test-subj="includeManagedSwitch" - /> - - - - - - - - - + {renderHeader()} = ({ > {name} - {isManagedByIngestManager(dataStream) ? ( - -   - - - - - - - ) : null} + ); }, diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/components/index.ts b/x-pack/plugins/index_management/public/application/sections/home/template_list/components/index.ts index 3954ce04ca0b53..cccdcaf9389bdc 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/components/index.ts +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/components/index.ts @@ -4,6 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './filter_list_button'; - export * from './template_type_indicator'; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx index 266003c5f89495..6edabbb2867a22 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx @@ -36,7 +36,7 @@ import { getIsLegacyFromQueryParams } from '../../../lib/index_templates'; import { TemplateTable } from './template_table'; import { TemplateDetails } from './template_details'; import { LegacyTemplateTable } from './legacy_templates/template_table'; -import { FilterListButton, Filters } from './components'; +import { FilterListButton, Filters } from '../components'; import { attemptToURIDecode } from '../../../../shared_imports'; type FilterName = 'managed' | 'cloudManaged' | 'system'; diff --git a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts index d19383d892cbd4..4124d8e897b5b9 100644 --- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts @@ -65,12 +65,24 @@ const enhanceDataStreams = ({ }); }; +const getDataStreams = (client: ElasticsearchClient, name = '*') => { + // TODO update when elasticsearch client has update requestParams for 'indices.getDataStream' + return client.transport.request({ + path: `/_data_stream/${encodeURIComponent(name)}`, + method: 'GET', + querystring: { + expand_wildcards: 'all', + }, + }); +}; + const getDataStreamsStats = (client: ElasticsearchClient, name = '*') => { return client.transport.request({ path: `/_data_stream/${encodeURIComponent(name)}/_stats`, method: 'GET', querystring: { human: true, + expand_wildcards: 'all', }, }); }; @@ -107,7 +119,7 @@ export function registerGetAllRoute({ try { let { body: { data_streams: dataStreams }, - } = await asCurrentUser.indices.getDataStream(); + } = await getDataStreams(asCurrentUser); let dataStreamsStats; let dataStreamsPrivileges; @@ -165,7 +177,7 @@ export function registerGetOneRoute({ body: { data_streams: dataStreamsStats }, }, ] = await Promise.all([ - asCurrentUser.indices.getDataStream({ name }), + getDataStreams(asCurrentUser, name), getDataStreamsStats(asCurrentUser, name), ]); diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index 6cf1a40a4d5a13..a8999fd065e752 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -14,6 +14,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; // @ts-ignore import { API_BASE_PATH } from './constants'; +import { DataStream } from '../../../../../plugins/index_management/common'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -66,32 +67,41 @@ export default function ({ getService }: FtrProviderContext) { before(async () => await createDataStream(testDataStreamName)); after(async () => await deleteDataStream(testDataStreamName)); - it('returns an array of all data streams', async () => { + it('returns an array of data streams', async () => { const { body: dataStreams } = await supertest .get(`${API_BASE_PATH}/data_streams`) .set('kbn-xsrf', 'xxx') .expect(200); + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + // ES determines these values so we'll just echo them back. - const { name: indexName, uuid } = dataStreams[0].indices[0]; - expect(dataStreams).to.eql([ - { - name: testDataStreamName, - privileges: { - delete_index: true, - }, - timeStampField: { name: '@timestamp' }, - indices: [ - { - name: indexName, - uuid, - }, - ], - generation: 1, - health: 'yellow', - indexTemplateName: testDataStreamName, + const { name: indexName, uuid } = testDataStream!.indices[0]; + + expect(testDataStream).to.eql({ + name: testDataStreamName, + privileges: { + delete_index: true, }, - ]); + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + uuid, + }, + ], + generation: 1, + health: 'yellow', + indexTemplateName: testDataStreamName, + hidden: false, + }); }); it('includes stats when provided the includeStats query parameter', async () => { @@ -100,12 +110,21 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'xxx') .expect(200); + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + // ES determines these values so we'll just echo them back. - const { name: indexName, uuid } = dataStreams[0].indices[0]; - const { storageSize, ...dataStreamWithoutStorageSize } = dataStreams[0]; + + const { name: indexName, uuid } = testDataStream!.indices[0]; + const { storageSize, ...dataStreamWithoutStorageSize } = testDataStream!; assertDataStreamStorageSizeExists(storageSize); - expect(dataStreams.length).to.be(1); expect(dataStreamWithoutStorageSize).to.eql({ name: testDataStreamName, privileges: { @@ -122,6 +141,7 @@ export default function ({ getService }: FtrProviderContext) { health: 'yellow', indexTemplateName: testDataStreamName, maxTimeStamp: 0, + hidden: false, }); }); @@ -152,6 +172,7 @@ export default function ({ getService }: FtrProviderContext) { health: 'yellow', indexTemplateName: testDataStreamName, maxTimeStamp: 0, + hidden: false, }); }); });