From b34b9f12c38b99eb1f23675e22fd9f5bd4ab7156 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 21 Jun 2024 13:27:53 +0200 Subject: [PATCH 01/28] Fix refresh table breaking for summary --- x-pack/test/functional/page_objects/dataset_quality.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index d9b274e1585367..57d4e43d2d58b0 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -219,6 +219,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async refreshTable() { + await testSubjects.waitForEnabled( + testSubjectSelectors.datasetQualityFiltersContainer, + 10 * 1000 + ); const filtersContainer = await testSubjects.find( testSubjectSelectors.datasetQualityFiltersContainer ); From d1fdaebaa8fb5b6dbfaccef41b51f4364daf4444 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 21 Jun 2024 13:30:45 +0200 Subject: [PATCH 02/28] Add timeout to find --- x-pack/test/functional/page_objects/dataset_quality.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 57d4e43d2d58b0..490118ee7ebec9 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -219,13 +219,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async refreshTable() { - await testSubjects.waitForEnabled( + const filtersContainer = await testSubjects.find( testSubjectSelectors.datasetQualityFiltersContainer, 10 * 1000 ); - const filtersContainer = await testSubjects.find( - testSubjectSelectors.datasetQualityFiltersContainer - ); const refreshButton = await filtersContainer.findByTestSubject( testSubjectSelectors.superDatePickerApplyTimeButton ); From db7d63b8387626d559bd14aebd9d000112946c67 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 21 Jun 2024 13:37:32 +0200 Subject: [PATCH 03/28] Revert "Add timeout to find" This reverts commit d1fdaebaa8fb5b6dbfaccef41b51f4364daf4444. --- x-pack/test/functional/page_objects/dataset_quality.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 490118ee7ebec9..57d4e43d2d58b0 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -219,10 +219,13 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async refreshTable() { - const filtersContainer = await testSubjects.find( + await testSubjects.waitForEnabled( testSubjectSelectors.datasetQualityFiltersContainer, 10 * 1000 ); + const filtersContainer = await testSubjects.find( + testSubjectSelectors.datasetQualityFiltersContainer + ); const refreshButton = await filtersContainer.findByTestSubject( testSubjectSelectors.superDatePickerApplyTimeButton ); From c3a2774031af9678857111aa1180476c22862e8c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 21 Jun 2024 13:37:32 +0200 Subject: [PATCH 04/28] Revert "Fix refresh table breaking for summary" This reverts commit b34b9f12c38b99eb1f23675e22fd9f5bd4ab7156. --- x-pack/test/functional/page_objects/dataset_quality.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 57d4e43d2d58b0..d9b274e1585367 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -219,10 +219,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async refreshTable() { - await testSubjects.waitForEnabled( - testSubjectSelectors.datasetQualityFiltersContainer, - 10 * 1000 - ); const filtersContainer = await testSubjects.find( testSubjectSelectors.datasetQualityFiltersContainer ); From 19015ad644469c6ccadcf31f3070ba19d4374a5c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 9 Oct 2024 12:17:31 +0200 Subject: [PATCH 05/28] Add code to implement Fix It flow for Field Limits --- .../dataset_quality/common/api_types.ts | 21 +++ .../common/data_stream_details/types.ts | 7 + .../dataset_quality/common/translations.ts | 144 +++++++++++++++ .../common/utils/component_template_name.ts | 18 ++ .../degraded_field_flyout/field_info.tsx | 2 +- .../degraded_field_flyout/index.tsx | 3 + .../field_limit_documentation_link.tsx | 30 ++++ .../field_limit/field_mapping_limit.tsx | 104 +++++++++++ .../increase_field_mapping_limit.tsx | 80 +++++++++ .../field_limit/message_callout.tsx | 101 +++++++++++ .../possible_mitigations/index.tsx | 33 ++++ .../possible_mitigations/manual/index.tsx | 113 ++++++++++++ .../possible_mitigations/title.tsx | 37 ++++ .../degraded_fields/degraded_fields.tsx | 1 + .../public/hooks/use_degraded_fields.ts | 42 +++-- .../data_stream_details_client.ts | 49 ++++++ .../services/data_stream_details/types.ts | 11 +- .../notifications.ts | 25 +++ .../state_machine.ts | 164 +++++++++++++----- .../types.ts | 48 ++++- .../get_data_stream_details/index.ts | 2 + .../get_datastream_settings.ts | 19 +- .../get_degraded_field_analysis/index.ts | 9 +- .../server/routes/data_streams/routes.ts | 61 +++++++ .../data_streams/update_field_limit/index.ts | 55 ++++++ .../update_component_template.ts | 53 ++++++ .../update_settings_last_backing_index.ts | 41 +++++ .../utils/create_dataset_quality_es_client.ts | 16 ++ 28 files changed, 1225 insertions(+), 64 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index bfbb2bc1cd5d1a..68f3e07bd16aee 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -134,14 +134,35 @@ export const degradedFieldAnalysisRt = rt.intersection([ type: rt.string, ignore_above: rt.number, }), + defaultPipeline: rt.string, }), ]); export type DegradedFieldAnalysis = rt.TypeOf; +export const updateFieldLimitResponseRt = rt.intersection([ + rt.type({ + isComponentTemplateUpdated: rt.union([rt.boolean, rt.undefined]), + isLatestBackingIndexUpdated: rt.union([rt.boolean, rt.undefined]), + customComponentTemplateName: rt.string, + }), + rt.partial({ + error: rt.string, + }), +]); + +export type UpdateFieldLimitResponse = rt.TypeOf; + +export const dataStreamRolloverResponseRt = rt.type({ + acknowledged: rt.boolean, +}); + +export type DataStreamRolloverResponse = rt.TypeOf; + export const dataStreamSettingsRt = rt.intersection([ rt.type({ lastBackingIndexName: rt.string, + indexTemplate: rt.string, }), rt.partial({ createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index 66b7567a2b60c0..d88cab5dd01a97 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -14,3 +14,10 @@ export interface AnalyzeDegradedFieldsParams { lastBackingIndex: string; degradedField: string; } + +export interface UpdateFieldLimitParams { + dataStream: string; + newFieldLimit: number; + indexTemplate: string; + lastBackingIndex: string; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index e5b660b31de108..0275976fa8afdc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -500,3 +500,147 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate 'This issue was detected in an older version of the dataset, but not in the most recent version.', } ); + +export const possibleMitigationTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigationTitle', + { + defaultMessage: 'Possible mitigation', + } +); + +export const increaseFieldMappingLimitTitle = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.increaseFieldMappingLimitTitle', + { + defaultMessage: 'Increase field mapping limit', + } +); + +export const fieldLimitMitigationDescriptionText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationDescription', + { + defaultMessage: + 'The field mapping limit sets the maximum number of fields in an index. When exceeded,additional fields are ignored. To prevent this, increase your field mapping limit.', + } +); + +export const fieldLimitMitigationConsiderationText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations', + { + defaultMessage: 'Before changing the field limit, consider the following:', + } +); + +export const fieldLimitMitigationConsiderationText1 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations1', + { + defaultMessage: 'Increasing the field limit could slow cluster performance.', + } +); + +export const fieldLimitMitigationConsiderationText2 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations2', + { + defaultMessage: 'Increasing the field limit also resolves field limit issues for other fields.', + } +); + +export const fieldLimitMitigationConsiderationText3 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations3', + { + defaultMessage: + 'This change applies to the [name] component template and affects all namespaces in the template.', + } +); + +export const fieldLimitMitigationConsiderationText4 = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationConsiderations4', + { + defaultMessage: + 'You need to roll over affected data streams to apply mapping changes to component templates.', + } +); + +export const fieldLimitMitigationCurrentLimitLabelText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationCurrentLimitLabelText', + { + defaultMessage: 'Current limit', + } +); + +export const fieldLimitMitigationNewLimitButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitButtonText', + { + defaultMessage: 'New limit', + } +); + +export const fieldLimitMitigationNewLimitPlaceholderText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationNewLimitPlaceholderText', + { + defaultMessage: 'New field limit', + } +); + +export const fieldLimitMitigationApplyButtonText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationApplyButtonText', + { + defaultMessage: 'Apply', + } +); + +export const otherMitigationsCustomComponentTemplate = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomComponentTemplate', + { + defaultMessage: 'Add or edit custom component template', + } +); + +export const otherMitigationsCustomIngestPipeline = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomIngestPipeline', + { + defaultMessage: 'Add or edit custom ingest pipeline', + } +); + +export const fieldLimitMitigationOfficialDocumentation = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationOfficialDocumentation', + { + defaultMessage: 'Documentation', + } +); + +export const fieldLimitMitigationSuccessMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessMessage', + { + defaultMessage: 'New limit set!', + } +); + +export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationSuccessComponentTemplateLinkText', + { + defaultMessage: 'See component template', + } +); + +export const fieldLimitMitigationFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage', + { + defaultMessage: 'Sorry, the request failed partially', + } +); + +export const fieldLimitMitigationFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription', + { + defaultMessage: + 'The component template was successfully updated with the new limit, but we failed to updated the last backing index. Please do a rollover for the changes to be applicable on new data', + } +); + +export const fieldLimitMitigationRolloverButton = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationRolloverButton', + { + defaultMessage: 'Rollover', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts new file mode 100644 index 00000000000000..0be7b84137c835 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/component_template_name.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * There are index templates like this metrics-apm.service_transaction.10m@template which exists. + * Hence this @ needs to be removed to derive the custom component template name. + */ +export function getComponentTemplatePrefixFromIndexTemplate(indexTemplate: string) { + if (indexTemplate.includes('@')) { + return indexTemplate.split('@')[0]; + } + + return indexTemplate; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx index 1e6bda781d733b..bf9a6c2d13c547 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx @@ -125,7 +125,7 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) )} - {!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayValues && ( + {!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayIgnoredValuesAndLimit && ( <> + + {isUserViewingTheIssueOnLatestBackingIndex && } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx new file mode 100644 index 00000000000000..3152c906017264 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiLink } from '@elastic/eui'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations'; + +export function FieldLimitDocLink() { + const { + services: { docLinks }, + } = useKibanaContextForPlugin(); + + return ( + + + {fieldLimitMitigationOfficialDocumentation} + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx new file mode 100644 index 00000000000000..2b8bcb9e313212 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { + EuiAccordion, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiPanel, + EuiSpacer, + EuiText, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + degradedFieldCurrentFieldLimitColumnName, + fieldLimitMitigationConsiderationText, + fieldLimitMitigationConsiderationText1, + fieldLimitMitigationConsiderationText2, + fieldLimitMitigationConsiderationText3, + fieldLimitMitigationConsiderationText4, + fieldLimitMitigationDescriptionText, + increaseFieldMappingLimitTitle, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; +import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit'; +import { FieldLimitDocLink } from './field_limit_documentation_link'; +import { MessageCallout } from './message_callout'; + +export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) { + const accordionId = useGeneratedHtmlId({ + prefix: increaseFieldMappingLimitTitle, + }); + + const { degradedFieldAnalysis } = useDegradedFields(); + + const accordionTitle = ( + +
{increaseFieldMappingLimitTitle}
+ + ); + + return ( + + + +

{fieldLimitMitigationDescriptionText}

+ + + {degradedFieldAnalysis?.isFieldLimitIssue && ( + + + + {degradedFieldCurrentFieldLimitColumnName} + + + + {degradedFieldAnalysis.totalFieldLimit} + + + )} + + +

{fieldLimitMitigationConsiderationText}

+ +
    +
  • {fieldLimitMitigationConsiderationText1}
    • +
  • {fieldLimitMitigationConsiderationText2}
    • +
  • {fieldLimitMitigationConsiderationText3}
    • +
  • {fieldLimitMitigationConsiderationText4}
    • +
+ + + {isIntegration && ( + <> + + + + + + )} + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx new file mode 100644 index 00000000000000..5ab701776be5a0 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiFieldText, + EuiFormRow, + EuiButton, + EuiFieldNumber, +} from '@elastic/eui'; +import { + fieldLimitMitigationApplyButtonText, + fieldLimitMitigationCurrentLimitLabelText, + fieldLimitMitigationNewLimitButtonText, + fieldLimitMitigationNewLimitPlaceholderText, +} from '../../../../../../common/translations'; +import { useDegradedFields } from '../../../../../hooks'; + +export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit: number }) { + // Propose the user a 30% increase over the current limit + const proposedNewLimit = Math.round(totalFieldLimit * 1.3); + const [newFieldLimit, setNewFieldLimit] = useState(proposedNewLimit); + const [isInvalid, setIsInvalid] = useState(false); + const { updateNewFieldLimit, isSavingNewFieldLimitInProgress } = useDegradedFields(); + + const validateNewLimit = (newLimit: string) => { + const parsedLimit = parseInt(newLimit, 10); + setNewFieldLimit(parsedLimit); + if (totalFieldLimit > parsedLimit) { + setIsInvalid(true); + } else { + setIsInvalid(false); + } + }; + + return ( + + + + + + + + + validateNewLimit(e.target.value)} + aria-label={fieldLimitMitigationNewLimitPlaceholderText} + isInvalid={isInvalid} + min={totalFieldLimit + 1} + /> + + + + + updateNewFieldLimit(newFieldLimit)} + isLoading={isSavingNewFieldLimitInProgress} + > + {fieldLimitMitigationApplyButtonText} + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx new file mode 100644 index 00000000000000..4993fb7a148f96 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { EuiButton, EuiCallOut, EuiFlexItem } from '@elastic/eui'; +import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import { + fieldLimitMitigationFailedMessage, + fieldLimitMitigationFailedMessageDescription, + fieldLimitMitigationRolloverButton, + fieldLimitMitigationSuccessComponentTemplateLinkText, + fieldLimitMitigationSuccessMessage, +} from '../../../../../../common/translations'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { useKibanaContextForPlugin } from '../../../../../utils'; + +export function MessageCallout() { + const { isSavingNewFieldLimitInProgress, newFieldLimitResult } = useDegradedFields(); + const { isComponentTemplateUpdated, isLatestBackingIndexUpdated } = newFieldLimitResult ?? {}; + const isSuccess = Boolean(isComponentTemplateUpdated) && Boolean(isLatestBackingIndexUpdated); + const isPartialSuccess = + Boolean(isComponentTemplateUpdated) && !Boolean(isLatestBackingIndexUpdated); + + if (!isSavingNewFieldLimitInProgress && isSuccess) { + return ; + } + + if (!isSavingNewFieldLimitInProgress && isPartialSuccess) { + return ; + } + + return null; +} + +export function SuccessCallout() { + const { + services: { application }, + } = useKibanaContextForPlugin(); + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); + const { name } = datasetDetails; + + const onClickHandler = useCallback(async () => { + await application.navigateToApp(MANAGEMENT_APP_ID, { + path: `/data/index_management/component_templates/${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + openInNewTab: true, + }); + }, [application, dataStreamSettings?.indexTemplate, name]); + + return ( + + + + {fieldLimitMitigationSuccessComponentTemplateLinkText} + + + + ); +} + +export function ManualRolloverCallout() { + const { triggerRollover } = useDegradedFields(); + return ( + + +

{fieldLimitMitigationFailedMessageDescription}

+ + {fieldLimitMitigationRolloverButton} + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx new file mode 100644 index 00000000000000..00d481e058695e --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { ManualMitigations } from './manual'; +import { FieldMappingLimit } from './field_limit/field_mapping_limit'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks'; +import { PossibleMitigationTitle } from './title'; + +export function PossibleMitigations() { + const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields(); + const { integrationDetails } = useDatasetQualityDetailsState(); + const isIntegration = Boolean(integrationDetails?.integration); + + return ( +
+ + + {!isAnalysisInProgress && degradedFieldAnalysis?.isFieldLimitIssue && ( + <> + + + + )} + {!isAnalysisInProgress && } +
+ ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx new file mode 100644 index 00000000000000..c16a7fc6303bf9 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiLink, + EuiPanel, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { + otherMitigationsCustomComponentTemplate, + otherMitigationsCustomIngestPipeline, +} from '../../../../../../common/translations'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; + +export function ManualMitigations() { + return ( + <> + + + + + ); +} + +function EditComponentTemplate() { + const { + services: { application }, + } = useKibanaContextForPlugin(); + + const { dataStreamSettings, integrationDetails, datasetDetails } = + useDatasetQualityDetailsState(); + const { name } = datasetDetails; + const isIntegration = !!integrationDetails?.integration; + + const onClickHandler = useCallback(async () => { + await application.navigateToApp(MANAGEMENT_APP_ID, { + path: isIntegration + ? `/data/index_management/component_templates/${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom` + : `/data/index_management/templates/${dataStreamSettings?.indexTemplate}`, + openInNewTab: true, + }); + }, [application, dataStreamSettings?.indexTemplate, isIntegration, name]); + + return ( + + + + + + + + +

{otherMitigationsCustomComponentTemplate}

+ + + + + + ); +} + +function EditPipeline() { + const { + services: { application }, + } = useKibanaContextForPlugin(); + const { degradedFieldAnalysis } = useDegradedFields(); + + const onClickHandler = async () => { + await application.navigateToApp(MANAGEMENT_APP_ID, { + path: `/ingest/ingest_pipelines/?pipeline=${degradedFieldAnalysis?.defaultPipeline}`, + openInNewTab: true, + }); + }; + + return ( + + + + + + + + +

{otherMitigationsCustomIngestPipeline}

+ + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx new file mode 100644 index 00000000000000..4a7eb22225ab8d --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiBetaBadge, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiTitle } from '@elastic/eui'; + +import { + overviewQualityIssuesAccordionTechPreviewBadge, + possibleMitigationTitle, +} from '../../../../../common/translations'; + +export function PossibleMitigationTitle() { + return ( + + + + + + +

{possibleMitigationTitle}

+ + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx index b33bd11dbe3a67..0cdc460cd56dca 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx @@ -45,6 +45,7 @@ export function DegradedFields() { aria-describedby={toggleTextSwitchId} compressed data-test-subj="datasetQualityDetailsOverviewDegradedFieldToggleSwitch" + css={{ marginRight: '5px' }} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 78ad0e53dd5e2c..248e9d1189a92a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -104,13 +104,13 @@ export function useDegradedFields() { }, [service]); const degradedFieldValues = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.ignoredValues.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.ignoredValues.done') ? state.context.degradedFieldValues : undefined ); const degradedFieldAnalysis = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.analyze.done') + state.matches('initializing.degradedFieldFlyout.open.initialized.analyze.done') ? state.context.degradedFieldAnalysis : undefined ); @@ -127,8 +127,7 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldLimitExceeded, tooltipContent: degradedFieldCauseFieldLimitExceededTooltip, - shouldDisplayMitigation: true, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, }; } @@ -143,8 +142,7 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldIgnored, tooltipContent: degradedFieldCauseFieldIgnoredTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: true, + shouldDisplayIgnoredValuesAndLimit: true, }; } } @@ -153,19 +151,39 @@ export function useDegradedFields() { return { potentialCause: degradedFieldCauseFieldMalformed, tooltipContent: degradedFieldCauseFieldMalformedTooltip, - shouldDisplayMitigation: false, - shouldDisplayValues: false, + shouldDisplayIgnoredValuesAndLimit: false, }; }, [degradedFieldAnalysis, degradedFieldValues]); const isDegradedFieldsValueLoading = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.ignoredValues.fetching'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + ); }); const isAnalysisInProgress = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.analyze.fetching'); + return state.matches('initializing.degradedFieldFlyout.open.initialized.analyze.fetching'); }); + const updateNewFieldLimit = useCallback( + (newFieldLimit: number) => { + service.send({ type: 'SET_NEW_FIELD_LIMIT', newFieldLimit }); + }, + [service] + ); + + const isSavingNewFieldLimitInProgress = useSelector(service, (state) => { + return state.matches('initializing.mitigations.savingNewFieldLimit'); + }); + + const newFieldLimitResult = useSelector(service, (state) => + state.matches('initializing.mitigations.done') ? state.context.fieldLimitResponse : undefined + ); + + const triggerRollover = useCallback(() => { + service.send('ROLLOVER_DATA_STREAM'); + }, [service]); + return { isDegradedFieldsLoading, pagination, @@ -185,5 +203,9 @@ export function useDegradedFields() { toggleCurrentQualityIssues, showCurrentQualityIssues, expandedRenderedItem, + updateNewFieldLimit, + isSavingNewFieldLimitInProgress, + newFieldLimitResult, + triggerRollover, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 9175d06e105b49..2ee1f888389f01 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,6 +8,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + DataStreamRolloverResponse, + dataStreamRolloverResponseRt, DegradedFieldAnalysis, degradedFieldAnalysisRt, DegradedFieldValues, @@ -19,6 +21,8 @@ import { IntegrationDashboardsResponse, integrationDashboardsRT, IntegrationResponse, + UpdateFieldLimitResponse, + updateFieldLimitResponseRt, } from '../../../common/api_types'; import { DataStreamDetails, @@ -37,6 +41,7 @@ import { Integration } from '../../../common/data_streams_stats/integration'; import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; import { DatasetQualityError } from '../../../common/errors'; @@ -196,4 +201,48 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { new DatasetQualityError(`Failed to decode the analysis response: ${message}`) )(response); } + + public async setNewFieldLimit({ + dataStream, + newFieldLimit, + indexTemplate, + lastBackingIndex, + }: UpdateFieldLimitParams): Promise { + const response = await this.http + .put( + `/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`, + { body: JSON.stringify({ newFieldLimit, indexTemplate, lastBackingIndex }) } + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to set new Limit": ${error}`, error); + }); + + const decodedResponse = decodeOrThrow( + updateFieldLimitResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode setting of new limit response: ${message}"`) + )(response); + + return decodedResponse; + } + + public async rolloverDataStream({ + dataStream, + }: { + dataStream: string; + }): Promise { + const response = await this.http + .post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to rollover datastream": ${error}`, error); + }); + + return decodeOrThrow( + dataStreamRolloverResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode rollover response: ${message}"`) + )(response); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index a2f7db99e5af1d..6eac8bd732840d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -20,8 +20,15 @@ import { import { AnalyzeDegradedFieldsParams, GetDataStreamIntegrationParams, + UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; -import { Dashboard, DegradedFieldAnalysis, DegradedFieldValues } from '../../../common/api_types'; +import { + Dashboard, + DataStreamRolloverResponse, + DegradedFieldAnalysis, + DegradedFieldValues, + UpdateFieldLimitResponse, +} from '../../../common/api_types'; export type DataStreamDetailsServiceSetup = void; @@ -47,4 +54,6 @@ export interface IDataStreamDetailsClient { params: GetDataStreamIntegrationParams ): Promise; analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; + setNewFieldLimit(params: UpdateFieldLimitParams): Promise; + rolloverDataStream(params: { dataStream: string }): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts index b501fd02bdcf3f..f5fdd063492a3f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts @@ -59,3 +59,28 @@ export const fetchDataStreamIntegrationFailedNotifier = ( text: error.message, }); }; + +export const updateFieldLimitFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.updateFieldLimitFailed', { + defaultMessage: "We couldn't update the field limit.", + }), + text: error.message, + }); +}; + +export const rolloverDataStreamFailedNotifier = ( + toasts: IToasts, + error: Error, + dataStream: string +) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.rolloverDataStreamFailed', { + defaultMessage: "We couldn't rollover the data stream: {dataStream}.", + values: { + dataStream, + }, + }), + text: error.message, + }); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 352aff140c2752..15aba2b00ea15f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -25,6 +25,7 @@ import { DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications'; import { @@ -33,6 +34,8 @@ import { fetchDataStreamSettingsFailedNotifier, fetchDataStreamIntegrationFailedNotifier, fetchIntegrationDashboardsFailedNotifier, + updateFieldLimitFailedNotifier, + rolloverDataStreamFailedNotifier, } from './notifications'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -189,10 +192,6 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, done: { on: { - UPDATE_TIME_RANGE: { - target: 'fetching', - actions: ['resetDegradedFieldPageAndRowsPerPage'], - }, UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { target: 'done', actions: ['storeDegradedFieldTableOptions'], @@ -200,7 +199,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( OPEN_DEGRADED_FIELD_FLYOUT: { target: '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', - actions: ['storeExpandedDegradedField'], + actions: [ + 'storeExpandedDegradedField', + 'resetFieldLimitServerResponse', + ], }, TOGGLE_CURRENT_QUALITY_ISSUES: { target: 'fetching', @@ -282,48 +284,53 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, open: { - type: 'parallel', + initial: 'initialized', states: { - ignoredValues: { - initial: 'fetching', + initialized: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDegradedFieldValues', - onDone: { - target: 'done', - actions: ['storeDegradedFieldValues'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', + ignoredValues: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldValues', + onDone: { + target: 'done', + actions: ['storeDegradedFieldValues'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], }, - { - target: 'done', - }, - ], + }, + done: {}, }, }, - done: {}, - }, - }, - analyze: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'analyzeDegradedField', - onDone: { - target: 'done', - actions: ['storeDegradedFieldAnalysis'], - }, - onError: { - target: 'done', + analyze: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'analyzeDegradedField', + onDone: { + target: 'done', + actions: ['storeDegradedFieldAnalysis'], + }, + onError: { + target: 'done', + }, + }, }, + done: {}, }, }, - done: {}, }, }, }, @@ -336,6 +343,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( target: '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', }, + SET_NEW_FIELD_LIMIT: { + target: + '#DatasetQualityDetailsController.initializing.mitigations.savingNewFieldLimit', + actions: 'storeNewFieldLimit', + }, + ROLLOVER_DATA_STREAM: { + target: '#DatasetQualityDetailsController.initializing.mitigations.rollover', + }, }, }, closed: { @@ -361,6 +376,39 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, }, + mitigations: { + initial: 'pending', + states: { + pending: {}, + savingNewFieldLimit: { + invoke: { + src: 'saveNewFieldLimit', + onDone: { + target: 'done', + actions: 'storeNewFieldLimitResponse', + }, + onError: { + target: 'done', + actions: 'notifySaveNewFieldLimitError', + }, + }, + }, + rollover: { + invoke: { + src: 'rolloverDataStream', + onDone: { + target: 'done', + actions: ['raiseForceTimeRangeRefresh'], + }, + onError: { + target: 'done', + actions: 'notifySaveNewFieldLimitError', + }, + }, + }, + done: {}, + }, + }, }, }, indexNotFound: { @@ -482,6 +530,19 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( isIndexNotFoundError: true, }; }), + storeNewFieldLimit: assign((_, event) => { + return 'newFieldLimit' in event ? { newFieldLimit: event.newFieldLimit } : {}; + }), + storeNewFieldLimitResponse: assign( + (_, event: DoneInvokeEvent) => { + return 'data' in event ? { fieldLimitResponse: event.data } : {}; + } + ), + resetFieldLimitServerResponse: assign(() => ({ + newFieldLimit: undefined, + fieldLimitResponse: undefined, + })), + raiseForceTimeRangeRefresh: raise('UPDATE_TIME_RANGE'), }, guards: { checkIfActionForbidden: (context, event) => { @@ -552,6 +613,10 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ 'dataStreamSettings' in context ? context.dataStreamSettings?.integration : undefined; return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName); }, + notifySaveNewFieldLimitError: (_context, event: DoneInvokeEvent) => + updateFieldLimitFailedNotifier(toasts, event.data), + notifyRolloverDataStreamError: (context, event: DoneInvokeEvent) => + rolloverDataStreamFailedNotifier(toasts, event.data, context.dataStream), }, services: { checkDatasetIsAggregatable: (context) => { @@ -661,6 +726,27 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ return Promise.resolve(); }, + saveNewFieldLimit: (context) => { + if ( + 'newFieldLimit' in context && + context.newFieldLimit && + 'dataStreamSettings' in context + ) { + return dataStreamDetailsClient.setNewFieldLimit({ + dataStream: context.dataStream, + newFieldLimit: context.newFieldLimit, + indexTemplate: context.dataStreamSettings.indexTemplate, + lastBackingIndex: context.dataStreamSettings.lastBackingIndexName, + }); + } + + return Promise.resolve(); + }, + rolloverDataStream: (context) => { + return dataStreamDetailsClient.rolloverDataStream({ + dataStream: context.dataStream, + }); + }, }, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdf3bfa579e558..9f58c14cbcd94a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -10,12 +10,14 @@ import type { DegradedFieldSortField } from '../../hooks'; import { Dashboard, DataStreamDetails, + DataStreamRolloverResponse, DataStreamSettings, DegradedField, DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFieldLimitResponse, } from '../../../common/api_types'; import { TableCriteria, TimeRangeConfig } from '../../../common/types'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -48,6 +50,8 @@ export interface WithDefaultControllerState { integration?: Integration; expandedDegradedField?: string; isNonAggregatable?: boolean; + newFieldLimit?: number; + fieldLimitResponse?: UpdateFieldLimitResponse; } export interface WithDataStreamDetails { @@ -87,6 +91,14 @@ export interface WithDegradeFieldAnalysis { degradedFieldAnalysis: DegradedFieldAnalysis; } +export interface WithNewFieldLimit { + newFieldLimit: number; +} + +export interface WithNewFieldLimitResponse { + fieldLimitResponse: UpdateFieldLimitResponse; +} + export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' @@ -128,16 +140,16 @@ export type DatasetQualityDetailsControllerTypeState = } | { value: - | 'initializing.degradedFieldFlyout.open.ignoredValues.fetching' - | 'initializing.degradedFieldFlyout.open.analyze.fetching'; + | 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + | 'initializing.degradedFieldFlyout.open.initialized.analyze.fetching'; context: WithDefaultControllerState & WithDegradedFieldsData; } | { - value: 'initializing.degradedFieldFlyout.open.ignoredValues.done'; + value: 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues; } | { - value: 'initializing.degradedFieldFlyout.open.analyze.done'; + value: 'initializing.degradedFieldFlyout.open.initialized.analyze.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis; } | { @@ -147,6 +159,23 @@ export type DatasetQualityDetailsControllerTypeState = WithDegradedFieldValues & WithDegradeFieldAnalysis; } + | { + value: 'initializing.mitigations.savingNewFieldLimit'; + context: WithDefaultControllerState & + WithDegradedFieldsData & + WithDegradedFieldValues & + WithDegradeFieldAnalysis & + WithNewFieldLimit; + } + | { + value: 'initializing.mitigations.done'; + context: WithDefaultControllerState & + WithDegradedFieldsData & + WithDegradedFieldValues & + WithDegradeFieldAnalysis & + WithNewFieldLimit & + WithNewFieldLimitResponse; + } | { value: | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' @@ -188,6 +217,13 @@ export type DatasetQualityDetailsControllerEvent = type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA'; degraded_field_criteria: TableCriteria; } + | { + type: 'SET_NEW_FIELD_LIMIT'; + newFieldLimit: number; + } + | { + type: 'ROLLOVER_DATA_STREAM'; + } | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent @@ -197,4 +233,6 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent; + | DoneInvokeEvent + | DoneInvokeEvent + | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index fd117d65ac99dd..6debbfcf1b0999 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -39,12 +39,14 @@ export async function getDataStreamSettings({ const integration = dataStreamInfo?._meta?.package?.name; const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; + const indexTemplate = dataStreamInfo.template; return { createdOn, integration, datasetUserPrivileges, lastBackingIndexName: lastBackingIndex?.index_name, + indexTemplate, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts index 433086c0b3e522..cbaa637dc60bc0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts @@ -13,6 +13,7 @@ export interface DataStreamSettingResponse { totalFieldLimit: number; ignoreDynamicBeyondLimit?: boolean; ignoreMalformed?: boolean; + defaultPipeline?: string; } const DEFAULT_FIELD_LIMIT = 1000; @@ -28,16 +29,20 @@ export async function getDataStreamSettings({ lastBackingIndex: string; }): Promise { const settings = await datasetQualityESClient.settings({ index: dataStream }); - const indexSettings = settings[lastBackingIndex]?.settings?.index?.mapping; + const setting = settings[lastBackingIndex]?.settings; + const mappingsInsideSettings = setting?.index?.mapping; return { - nestedFieldLimit: indexSettings?.nested_fields?.limit - ? Number(indexSettings?.nested_fields?.limit) + nestedFieldLimit: mappingsInsideSettings?.nested_fields?.limit + ? Number(mappingsInsideSettings?.nested_fields?.limit) : DEFAULT_NESTED_FIELD_LIMIT, - totalFieldLimit: indexSettings?.total_fields?.limit - ? Number(indexSettings?.total_fields?.limit) + totalFieldLimit: mappingsInsideSettings?.total_fields?.limit + ? Number(mappingsInsideSettings?.total_fields?.limit) : DEFAULT_FIELD_LIMIT, - ignoreDynamicBeyondLimit: toBoolean(indexSettings?.total_fields?.ignore_dynamic_beyond_limit), - ignoreMalformed: toBoolean(indexSettings?.ignore_malformed), + ignoreDynamicBeyondLimit: toBoolean( + mappingsInsideSettings?.total_fields?.ignore_dynamic_beyond_limit + ), + ignoreMalformed: toBoolean(mappingsInsideSettings?.ignore_malformed), + defaultPipeline: setting?.index?.default_pipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts index a0e7606b475b2c..97ff0b124aae90 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts @@ -28,7 +28,13 @@ export async function analyzeDegradedField({ const [ { fieldCount, fieldPresent, fieldMapping }, - { nestedFieldLimit, totalFieldLimit, ignoreDynamicBeyondLimit, ignoreMalformed }, + { + nestedFieldLimit, + totalFieldLimit, + ignoreDynamicBeyondLimit, + ignoreMalformed, + defaultPipeline, + }, ] = await Promise.all([ getDataStreamMapping({ datasetQualityESClient, @@ -48,5 +54,6 @@ export async function analyzeDegradedField({ totalFieldLimit, ignoreMalformed, nestedFieldLimit, + defaultPipeline, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 047004d58a6a2d..65112ea7f7bab7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -16,6 +16,8 @@ import { DatasetUserPrivileges, DegradedFieldValues, DegradedFieldAnalysis, + UpdateFieldLimitResponse, + DataStreamRolloverResponse, } from '../../../common/api_types'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; @@ -29,6 +31,8 @@ import { getDegradedFields } from './get_degraded_fields'; import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; +import { updateFieldLimit } from './update_field_limit'; +import { createDatasetQualityESClient } from '../../utils'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -324,6 +328,61 @@ const analyzeDegradedFieldRoute = createDatasetQualityServerRoute({ }, }); +const updateFieldLimitRoute = createDatasetQualityServerRoute({ + endpoint: 'PUT /internal/dataset_quality/data_streams/{dataStream}/update_field_limit', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + body: t.type({ + newFieldLimit: t.number, + indexTemplate: t.string, + lastBackingIndex: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const updatedLimitResponse = await updateFieldLimit({ + esClient, + indexTemplate: params.body.indexTemplate, + lastBackingIndex: params.body.lastBackingIndex, + newFieldLimit: params.body.newFieldLimit, + }); + + return updatedLimitResponse; + }, +}); + +const rolloverDataStream = createDatasetQualityServerRoute({ + endpoint: 'POST /internal/dataset_quality/data_streams/{dataStream}/rollover', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { acknowledged } = await datasetQualityESClient.rollover({ + alias: params.path.dataStream, + }); + + return { acknowledged }; + }, +}); + export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, @@ -334,4 +393,6 @@ export const dataStreamsRouteRepository = { ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, ...analyzeDegradedFieldRoute, + ...updateFieldLimitRoute, + ...rolloverDataStream, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts new file mode 100644 index 00000000000000..dae04725e6a9bc --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { createDatasetQualityESClient } from '../../../utils'; +import { updateComponentTemplate } from './update_component_template'; +import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; +import { UpdateFieldLimitResponse } from '../../../../common/api_types'; + +export async function updateFieldLimit({ + esClient, + indexTemplate, + lastBackingIndex, + newFieldLimit, +}: { + esClient: ElasticsearchClient; + indexTemplate: string; + lastBackingIndex: string; + newFieldLimit: number; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const { + acknowledged: isComponentTemplateUpdated, + componentTemplateName, + error: errorUpdatingComponentTemplate, + } = await updateComponentTemplate({ datasetQualityESClient, indexTemplate, newFieldLimit }); + + if (!isComponentTemplateUpdated) { + return { + isComponentTemplateUpdated, + isLatestBackingIndexUpdated: false, + customComponentTemplateName: componentTemplateName, + error: errorUpdatingComponentTemplate, + }; + } + + const { acknowledged: isLatestBackingIndexUpdated, error: errorUpdatingBackingIndex } = + await updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex, + newFieldLimit, + }); + + return { + isComponentTemplateUpdated, + isLatestBackingIndexUpdated, + customComponentTemplateName: componentTemplateName, + error: errorUpdatingBackingIndex, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts new file mode 100644 index 00000000000000..0bf19410bd6ac6 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_component_template.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name'; + +interface UpdateComponentTemplateResponse { + acknowledged: boolean | undefined; + componentTemplateName: string; + error?: string; +} + +export async function updateComponentTemplate({ + datasetQualityESClient, + indexTemplate, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + indexTemplate: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + settings: { + 'index.mapping.total_fields.limit': newFieldLimit, + }, + }; + + const customComponentTemplateName = `${getComponentTemplatePrefixFromIndexTemplate( + indexTemplate + )}@custom`; + + try { + const { acknowledged } = await datasetQualityESClient.updateComponentTemplate({ + name: customComponentTemplateName, + template: newSettings, + }); + + return { + acknowledged, + componentTemplateName: customComponentTemplateName, + }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + componentTemplateName: customComponentTemplateName, + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts new file mode 100644 index 00000000000000..b98a3155475543 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/update_settings_last_backing_index.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; + +interface UpdateLastBackingIndexSettingsResponse { + acknowledged: boolean | undefined; + error?: string; +} + +export async function updateLastBackingIndexSettings({ + datasetQualityESClient, + lastBackingIndex, + newFieldLimit, +}: { + datasetQualityESClient: DatasetQualityESClient; + lastBackingIndex: string; + newFieldLimit: number; +}): Promise { + const newSettings = { + 'index.mapping.total_fields.limit': newFieldLimit, + }; + + try { + const { acknowledged } = await datasetQualityESClient.updateSettings({ + index: lastBackingIndex, + settings: newSettings, + }); + + return { acknowledged }; + } catch (error) { + return { + acknowledged: undefined, // acknowledge is undefined when the request fails + error: error.message, + }; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index baa2403690fd89..006d776779bf49 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -8,11 +8,16 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ElasticsearchClient } from '@kbn/core/server'; import { + ClusterPutComponentTemplateRequest, + ClusterPutComponentTemplateResponse, FieldCapsRequest, FieldCapsResponse, Indices, IndicesGetMappingResponse, IndicesGetSettingsResponse, + IndicesPutSettingsRequest, + IndicesPutSettingsResponse, + IndicesRolloverResponse, } from '@elastic/elasticsearch/lib/api/types'; type DatasetQualityESSearchParams = ESSearchRequest & { @@ -47,5 +52,16 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { async settings(params: { index: string }): Promise { return esClient.indices.getSettings(params); }, + async updateComponentTemplate( + params: ClusterPutComponentTemplateRequest + ): Promise { + return esClient.cluster.putComponentTemplate(params); + }, + async updateSettings(params: IndicesPutSettingsRequest): Promise { + return esClient.indices.putSettings(params); + }, + async rollover(params: { alias: string }): Promise { + return esClient.indices.rollover(params); + }, }; } From a5f32c611cd58d04c841ec68c221f9078be18fed Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 9 Oct 2024 19:39:26 +0200 Subject: [PATCH 06/28] Add code logic to show a warning message for ignore_malformed --- .../dataset_quality/common/translations.ts | 12 ++++++++-- .../degraded_field_flyout/index.tsx | 23 ++++++++++++++++++- .../public/hooks/use_degraded_fields.ts | 5 ++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 0275976fa8afdc..f24bf07502335e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -501,6 +501,14 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate } ); +export const degradedFieldPotentialCauseIgnoreMalformedWarning = i18n.translate( + 'xpack.datasetQuality.details.degradedField.potentialCause.ignoreMalformedWarning', + { + defaultMessage: + 'If you recently changed any settings, it is possible that this potential cause may not be valid any more. Please rollover to check.', + } +); + export const possibleMitigationTitle = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigationTitle', { @@ -626,7 +634,7 @@ export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.transla export const fieldLimitMitigationFailedMessage = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage', { - defaultMessage: 'Sorry, the request failed partially', + defaultMessage: 'Changes not applied to new data', } ); @@ -634,7 +642,7 @@ export const fieldLimitMitigationFailedMessageDescription = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription', { defaultMessage: - 'The component template was successfully updated with the new limit, but we failed to updated the last backing index. Please do a rollover for the changes to be applicable on new data', + 'The component template was successfully updated with the new field limit, but the changes were not applied to the most recent backing index. Perform a rollover to apply your changes to new data.', } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 502048499b5330..2a74646fdd30bb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -29,6 +29,7 @@ import { } from '../../../hooks'; import { degradedFieldMessageIssueDoesNotExistInLatestIndex, + degradedFieldPotentialCauseIgnoreMalformedWarning, discoverAriaText, fieldIgnoredText, logsExplorerAriaText, @@ -43,7 +44,13 @@ import { PossibleMitigations } from './possible_mitigations'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedFieldFlyout() { - const { closeDegradedFieldFlyout, expandedDegradedField, renderedItems } = useDegradedFields(); + const { + closeDegradedFieldFlyout, + expandedDegradedField, + renderedItems, + isAnalysisInProgress, + degradedFieldAnalysisResult, + } = useDegradedFields(); const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState(); const pushedFlyoutTitleId = useGeneratedHtmlId({ prefix: 'pushedFlyoutTitle', @@ -119,6 +126,20 @@ export default function DegradedFieldFlyout() { )} + {isUserViewingTheIssueOnLatestBackingIndex && + !isAnalysisInProgress && + degradedFieldAnalysisResult && + !degradedFieldAnalysisResult.identifiedUsingHeuristics && ( + <> + + + {degradedFieldPotentialCauseIgnoreMalformedWarning} + + + )} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 248e9d1189a92a..c6f2e3e761dc2d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -115,8 +115,6 @@ export function useDegradedFields() { : undefined ); - // This piece only cater field limit issue at the moment. - // In future this will cater the other 2 reasons as well const degradedFieldAnalysisResult = useMemo(() => { if (!degradedFieldAnalysis) { return undefined; @@ -128,6 +126,7 @@ export function useDegradedFields() { potentialCause: degradedFieldCauseFieldLimitExceeded, tooltipContent: degradedFieldCauseFieldLimitExceededTooltip, shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: true, }; } @@ -143,6 +142,7 @@ export function useDegradedFields() { potentialCause: degradedFieldCauseFieldIgnored, tooltipContent: degradedFieldCauseFieldIgnoredTooltip, shouldDisplayIgnoredValuesAndLimit: true, + identifiedUsingHeuristics: true, }; } } @@ -152,6 +152,7 @@ export function useDegradedFields() { potentialCause: degradedFieldCauseFieldMalformed, tooltipContent: degradedFieldCauseFieldMalformedTooltip, shouldDisplayIgnoredValuesAndLimit: false, + identifiedUsingHeuristics: false, // TODO: Add heuristics to identify ignore_malformed issues }; }, [degradedFieldAnalysis, degradedFieldValues]); From ebc986c181215b9111c5749e85838bcbf1387249 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 9 Oct 2024 19:55:14 +0200 Subject: [PATCH 07/28] Remove unwanted tests --- .../tests/data_streams/data_stream_settings.spec.ts | 7 ------- .../data_stream_settings.ts | 7 ------- 2 files changed, 14 deletions(-) diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts index 45f37b44983aa1..151aabbbc059f3 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts @@ -104,13 +104,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); }); - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApiAs('datasetQualityMonitorUser', nonExistentDataStream); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts index a132bc01c97207..410166447e3dd0 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts @@ -91,13 +91,6 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); }); - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApi(nonExistentDataStream, roleAuthc, internalReqHeader); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, From fd3df0178bc5fad1c3ddac83f7c2a5e0d04c4d1b Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 10 Oct 2024 11:18:34 +0200 Subject: [PATCH 08/28] Fix text for situation where previously field_limit existed --- .../dataset_quality/common/translations.ts | 8 -------- .../degraded_field_flyout/index.tsx | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index f24bf07502335e..d6d711a4e59ce1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -501,14 +501,6 @@ export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate } ); -export const degradedFieldPotentialCauseIgnoreMalformedWarning = i18n.translate( - 'xpack.datasetQuality.details.degradedField.potentialCause.ignoreMalformedWarning', - { - defaultMessage: - 'If you recently changed any settings, it is possible that this potential cause may not be valid any more. Please rollover to check.', - } -); - export const possibleMitigationTitle = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigationTitle', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 2a74646fdd30bb..3af49c6327a21d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -6,6 +6,7 @@ */ import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiBadge, EuiFlyout, @@ -20,6 +21,7 @@ import { EuiButtonIcon, EuiToolTip, } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { NavigationSource } from '../../../services/telemetry'; import { useDatasetDetailsRedirectLinkTelemetry, @@ -29,7 +31,6 @@ import { } from '../../../hooks'; import { degradedFieldMessageIssueDoesNotExistInLatestIndex, - degradedFieldPotentialCauseIgnoreMalformedWarning, discoverAriaText, fieldIgnoredText, logsExplorerAriaText, @@ -136,7 +137,20 @@ export default function DegradedFieldFlyout() { color="danger" data-test-subj="datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist" > - {degradedFieldPotentialCauseIgnoreMalformedWarning} + + {i18n.translate( + 'xpack.datasetQuality.degradedFieldFlyout.strong.fieldLimitLabel', + { defaultMessage: 'field limit' } + )} + + ), + }} + /> )} From 6875a73080782487512d78af4e50c64fb0d9afa7 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Thu, 10 Oct 2024 12:20:59 +0200 Subject: [PATCH 09/28] Update x-pack/plugins/observability_solution/dataset_quality/common/translations.ts Co-authored-by: Marco Antonio Ghiani --- .../dataset_quality/common/translations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index d6d711a4e59ce1..b88f837b4f65a3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -519,7 +519,7 @@ export const fieldLimitMitigationDescriptionText = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationDescription', { defaultMessage: - 'The field mapping limit sets the maximum number of fields in an index. When exceeded,additional fields are ignored. To prevent this, increase your field mapping limit.', + 'The field mapping limit sets the maximum number of fields in an index. When exceeded, additional fields are ignored. To prevent this, increase your field mapping limit.', } ); From f1ba3e7b641d0ab06eb607370bf920f0f3ea1dca Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 10 Oct 2024 14:54:48 +0200 Subject: [PATCH 10/28] Fix checktype issues --- .../dataset_quality_api_integration/data_stream_settings.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts index 410166447e3dd0..4a70cd1b2298c3 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts @@ -30,10 +30,6 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { const serviceName = 'my-service'; const hostName = 'synth-host'; - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - async function callApi( dataStream: string, roleAuthc: RoleCredentials, From 9fd87ffbfec3dff9211f84fc96affd9fe66ca17f Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 10 Oct 2024 15:36:05 +0200 Subject: [PATCH 11/28] Fix pipeline logic for integrations and non integrations --- .../dataset_quality/common/translations.ts | 7 +++ .../possible_mitigations/manual/index.tsx | 43 ++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index b88f837b4f65a3..0a6596a868f775 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -644,3 +644,10 @@ export const fieldLimitMitigationRolloverButton = i18n.translate( defaultMessage: 'Rollover', } ); + +export const manualMitigationCustomPipelineText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomPipelineText', + { + defaultMessage: 'Copy the name below to add or edit the custom pipeline', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index c16a7fc6303bf9..874849172c440c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -13,36 +13,40 @@ import { EuiLink, EuiPanel, EuiSpacer, + EuiText, EuiTitle, + EuiCopy, + EuiButtonEmpty, } from '@elastic/eui'; import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; import { useKibanaContextForPlugin } from '../../../../../utils'; import { + manualMitigationCustomPipelineText, otherMitigationsCustomComponentTemplate, otherMitigationsCustomIngestPipeline, } from '../../../../../../common/translations'; -import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; export function ManualMitigations() { + const { integrationDetails } = useDatasetQualityDetailsState(); + const isIntegration = !!integrationDetails?.integration; return ( <> - + - + ); } -function EditComponentTemplate() { +function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { const { services: { application }, } = useKibanaContextForPlugin(); - const { dataStreamSettings, integrationDetails, datasetDetails } = - useDatasetQualityDetailsState(); + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); const { name } = datasetDetails; - const isIntegration = !!integrationDetails?.integration; const onClickHandler = useCallback(async () => { await application.navigateToApp(MANAGEMENT_APP_ID, { @@ -77,15 +81,18 @@ function EditComponentTemplate() { ); } -function EditPipeline() { +function EditPipeline({ isIntegration }: { isIntegration: boolean }) { const { services: { application }, } = useKibanaContextForPlugin(); - const { degradedFieldAnalysis } = useDegradedFields(); + const { datasetDetails } = useDatasetQualityDetailsState(); + const { type, name } = datasetDetails; + + const copyText = isIntegration ? `${type}-${name}@custom` : `${type}@custom`; const onClickHandler = async () => { await application.navigateToApp(MANAGEMENT_APP_ID, { - path: `/ingest/ingest_pipelines/?pipeline=${degradedFieldAnalysis?.defaultPipeline}`, + path: '/ingest/ingest_pipelines/?pipeline', openInNewTab: true, }); }; @@ -108,6 +115,22 @@ function EditPipeline() { + + +

{manualMitigationCustomPipelineText}

+ + {(copy) => ( + + {copyText} + + )} + + ); } From 88b908c5c1eec72faa93aa900bb3eb036195093b Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 15 Oct 2024 15:48:40 +0200 Subject: [PATCH 12/28] Fix certain DOM level review comments --- .../field_limit_documentation_link.tsx | 21 ++++++------- .../field_limit/field_mapping_limit.tsx | 4 +-- .../possible_mitigations/manual/index.tsx | 31 +++++++------------ .../possible_mitigations/title.tsx | 31 ++++++++----------- .../utils/create_dataset_quality_es_client.ts | 18 +++++------ 5 files changed, 46 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx index 3152c906017264..2b69f5373f252c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -6,7 +6,7 @@ */ import React from 'react'; -import { EuiFlexGroup, EuiLink } from '@elastic/eui'; +import { EuiLink } from '@elastic/eui'; import { useKibanaContextForPlugin } from '../../../../../utils'; import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations'; @@ -16,15 +16,14 @@ export function FieldLimitDocLink() { } = useKibanaContextForPlugin(); return ( - - - {fieldLimitMitigationOfficialDocumentation} - - + + {fieldLimitMitigationOfficialDocumentation} + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx index 2b8bcb9e313212..57b3f6e8071c07 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -54,8 +54,8 @@ export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion" paddingSize="s" > - -

{fieldLimitMitigationDescriptionText}

+ + {fieldLimitMitigationDescriptionText} {degradedFieldAnalysis?.isFieldLimitIssue && ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index 874849172c440c..c0b0a4b1df6d04 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -8,7 +8,6 @@ import React, { useCallback } from 'react'; import { EuiFlexGroup, - EuiFlexItem, EuiIcon, EuiLink, EuiPanel, @@ -65,16 +64,13 @@ function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { data-test-subj="datasetQualityManualMitigationsCustomComponentTemplateLink" onClick={onClickHandler} target="_blank" + css={{ width: '100%' }} > - - - - - - -

{otherMitigationsCustomComponentTemplate}

- - + + + +

{otherMitigationsCustomComponentTemplate}

+ @@ -103,16 +99,13 @@ function EditPipeline({ isIntegration }: { isIntegration: boolean }) { data-test-subj="datasetQualityManualMitigationsPipelineLink" onClick={onClickHandler} target="_blank" + css={{ width: '100%' }} > - - - - - - -

{otherMitigationsCustomIngestPipeline}

- - + + + +

{otherMitigationsCustomIngestPipeline}

+ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx index 4a7eb22225ab8d..814f003feedd88 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx @@ -6,7 +6,7 @@ */ import React from 'react'; -import { EuiBetaBadge, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiTitle } from '@elastic/eui'; +import { EuiBetaBadge, EuiFlexGroup, EuiIcon, EuiTitle } from '@elastic/eui'; import { overviewQualityIssuesAccordionTechPreviewBadge, @@ -15,23 +15,18 @@ import { export function PossibleMitigationTitle() { return ( - - - - - - -

{possibleMitigationTitle}

- - - - - + + + +

{possibleMitigationTitle}

+ + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 006d776779bf49..8a78b4163da95c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -28,12 +28,12 @@ export type DatasetQualityESClient = ReturnType( + search( searchParams: TParams ): Promise> { return esClient.search(searchParams) as Promise; }, - async msearch( + msearch( index = {} as { index?: Indices }, searches: TParams[] ): Promise<{ @@ -43,24 +43,24 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { searches: searches.map((search) => [index, search]).flat(), }) as Promise; }, - async fieldCaps(params: FieldCapsRequest): Promise { - return esClient.fieldCaps(params) as Promise; + fieldCaps(params: FieldCapsRequest): Promise { + return esClient.fieldCaps(params); }, - async mappings(params: { index: string }): Promise { + mappings(params: { index: string }): Promise { return esClient.indices.getMapping(params); }, - async settings(params: { index: string }): Promise { + settings(params: { index: string }): Promise { return esClient.indices.getSettings(params); }, - async updateComponentTemplate( + updateComponentTemplate( params: ClusterPutComponentTemplateRequest ): Promise { return esClient.cluster.putComponentTemplate(params); }, - async updateSettings(params: IndicesPutSettingsRequest): Promise { + updateSettings(params: IndicesPutSettingsRequest): Promise { return esClient.indices.putSettings(params); }, - async rollover(params: { alias: string }): Promise { + rollover(params: { alias: string }): Promise { return esClient.indices.rollover(params); }, }; From 6c90898ebacdbd799945204db43de7dd1ef474b7 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 15 Oct 2024 16:55:24 +0200 Subject: [PATCH 13/28] Fix locator for Management plugin --- src/plugins/management/common/locator.ts | 42 +++++++++-- .../possible_mitigations/manual/index.tsx | 74 ++++++++----------- 2 files changed, 66 insertions(+), 50 deletions(-) diff --git a/src/plugins/management/common/locator.ts b/src/plugins/management/common/locator.ts index b5a87561534182..ca2cab0798a454 100644 --- a/src/plugins/management/common/locator.ts +++ b/src/plugins/management/common/locator.ts @@ -12,10 +12,22 @@ import { LocatorDefinition, LocatorPublic } from '@kbn/share-plugin/common'; import { MANAGEMENT_APP_LOCATOR } from '@kbn/deeplinks-management/constants'; import { MANAGEMENT_APP_ID } from './contants'; -export interface ManagementAppLocatorParams extends SerializableRecord { - sectionId: string; - appId?: string; -} +export type ManagementAppLocatorParams = SerializableRecord & + ( + | { + sectionId: string; + appId?: string; + } + | { + componentTemplate: string; + } + | { + indexTemplate: string; + } + | { + pipeline: string; + } + ); export type ManagementAppLocator = LocatorPublic; @@ -25,7 +37,7 @@ export class ManagementAppLocatorDefinition public readonly id = MANAGEMENT_APP_LOCATOR; public readonly getLocation = async (params: ManagementAppLocatorParams) => { - const path = `/${params.sectionId}${params.appId ? '/' + params.appId : ''}`; + const path = buildPathFromParams(params); return { app: MANAGEMENT_APP_ID, @@ -34,3 +46,23 @@ export class ManagementAppLocatorDefinition }; }; } + +const buildPathFromParams = (params: ManagementAppLocatorParams) => { + if (params.sectionId) { + return `/${params.sectionId}${params.appId ? '/' + params.appId : ''}`; + } + + if (params.indexTemplate) { + return `/data/index_management/templates/${params.indexTemplate}`; + } + + if (params.componentTemplate) { + return `/data/index_management/component_templates/${params.componentTemplate}`; + } + + if (params.pipeline) { + return `/ingest/ingest_pipelines/?pipeline=${params.pipeline}`; + } + + return '/'; +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index c0b0a4b1df6d04..2b133ad8045b2b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -5,19 +5,8 @@ * 2.0. */ -import React, { useCallback } from 'react'; -import { - EuiFlexGroup, - EuiIcon, - EuiLink, - EuiPanel, - EuiSpacer, - EuiText, - EuiTitle, - EuiCopy, - EuiButtonEmpty, -} from '@elastic/eui'; -import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import React from 'react'; +import { EuiLink, EuiPanel, EuiSpacer, EuiText, EuiCopy, EuiButtonEmpty } from '@elastic/eui'; import { useKibanaContextForPlugin } from '../../../../../utils'; import { manualMitigationCustomPipelineText, @@ -41,37 +30,36 @@ export function ManualMitigations() { function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { const { - services: { application }, + services: { + share: { + url: { locators }, + }, + }, } = useKibanaContextForPlugin(); const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); const { name } = datasetDetails; - const onClickHandler = useCallback(async () => { - await application.navigateToApp(MANAGEMENT_APP_ID, { - path: isIntegration - ? `/data/index_management/component_templates/${getComponentTemplatePrefixFromIndexTemplate( + const componentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl( + isIntegration + ? { + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( dataStreamSettings?.indexTemplate ?? name - )}@custom` - : `/data/index_management/templates/${dataStreamSettings?.indexTemplate}`, - openInNewTab: true, - }); - }, [application, dataStreamSettings?.indexTemplate, isIntegration, name]); + )}@custom`, + } + : { indexTemplate: dataStreamSettings?.indexTemplate } + ); return ( - - - -

{otherMitigationsCustomComponentTemplate}

- - + {otherMitigationsCustomComponentTemplate} ); @@ -79,34 +67,30 @@ function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { function EditPipeline({ isIntegration }: { isIntegration: boolean }) { const { - services: { application }, + services: { + share: { + url: { locators }, + }, + }, } = useKibanaContextForPlugin(); + const { datasetDetails } = useDatasetQualityDetailsState(); const { type, name } = datasetDetails; const copyText = isIntegration ? `${type}-${name}@custom` : `${type}@custom`; - const onClickHandler = async () => { - await application.navigateToApp(MANAGEMENT_APP_ID, { - path: '/ingest/ingest_pipelines/?pipeline', - openInNewTab: true, - }); - }; + const pipelineUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ pipeline: '' }); return ( - - - -

{otherMitigationsCustomIngestPipeline}

- - + {otherMitigationsCustomIngestPipeline} From c4f23047b9b2f8f54d55cd80c67f7f4ad681ea93 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 16 Oct 2024 11:12:06 +0200 Subject: [PATCH 14/28] Fix logic to move fix it flow to the server side --- .../dataset_quality/common/api_types.ts | 18 +++++++----------- .../common/data_stream_details/types.ts | 2 -- .../data_stream_details_client.ts | 4 +--- .../state_machine.ts | 8 +------- .../get_data_stream_details/index.ts | 2 +- .../server/routes/data_streams/routes.ts | 5 +---- .../data_streams/update_field_limit/index.ts | 9 +++++---- .../data_streams/data_stream_settings.spec.ts | 7 +++++++ .../data_stream_settings.ts | 11 +++++++++++ 9 files changed, 34 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 68f3e07bd16aee..903d7f06076634 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -159,17 +159,13 @@ export const dataStreamRolloverResponseRt = rt.type({ export type DataStreamRolloverResponse = rt.TypeOf; -export const dataStreamSettingsRt = rt.intersection([ - rt.type({ - lastBackingIndexName: rt.string, - indexTemplate: rt.string, - }), - rt.partial({ - createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless - integration: rt.string, - datasetUserPrivileges: datasetUserPrivilegesRt, - }), -]); +export const dataStreamSettingsRt = rt.partial({ + lastBackingIndexName: rt.string, + indexTemplate: rt.string, + createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless + integration: rt.string, + datasetUserPrivileges: datasetUserPrivilegesRt, +}); export type DataStreamSettings = rt.TypeOf; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index d88cab5dd01a97..ce74552b581b90 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -18,6 +18,4 @@ export interface AnalyzeDegradedFieldsParams { export interface UpdateFieldLimitParams { dataStream: string; newFieldLimit: number; - indexTemplate: string; - lastBackingIndex: string; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 2ee1f888389f01..40e146794c0c26 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -205,13 +205,11 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { public async setNewFieldLimit({ dataStream, newFieldLimit, - indexTemplate, - lastBackingIndex, }: UpdateFieldLimitParams): Promise { const response = await this.http .put( `/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`, - { body: JSON.stringify({ newFieldLimit, indexTemplate, lastBackingIndex }) } + { body: JSON.stringify({ newFieldLimit }) } ) .catch((error) => { throw new DatasetQualityError(`Failed to set new Limit": ${error}`, error); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 15aba2b00ea15f..ed573fdddb8d85 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -727,16 +727,10 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ return Promise.resolve(); }, saveNewFieldLimit: (context) => { - if ( - 'newFieldLimit' in context && - context.newFieldLimit && - 'dataStreamSettings' in context - ) { + if ('newFieldLimit' in context && context.newFieldLimit) { return dataStreamDetailsClient.setNewFieldLimit({ dataStream: context.dataStream, newFieldLimit: context.newFieldLimit, - indexTemplate: context.dataStreamSettings.indexTemplate, - lastBackingIndex: context.dataStreamSettings.lastBackingIndexName, }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index e570c206f2e26c..79180801c6c0ef 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -39,7 +39,7 @@ export async function getDataStreamSettings({ const integration = dataStreamInfo?._meta?.package?.name; const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; - const indexTemplate = dataStreamInfo.template; + const indexTemplate = dataStreamInfo?.template; return { createdOn, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 65112ea7f7bab7..41ba3ee8c72999 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -336,8 +336,6 @@ const updateFieldLimitRoute = createDatasetQualityServerRoute({ }), body: t.type({ newFieldLimit: t.number, - indexTemplate: t.string, - lastBackingIndex: t.string, }), }), options: { @@ -350,9 +348,8 @@ const updateFieldLimitRoute = createDatasetQualityServerRoute({ const updatedLimitResponse = await updateFieldLimit({ esClient, - indexTemplate: params.body.indexTemplate, - lastBackingIndex: params.body.lastBackingIndex, newFieldLimit: params.body.newFieldLimit, + dataStream: params.path.dataStream, }); return updatedLimitResponse; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts index dae04725e6a9bc..d7bad9d7fcbe87 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -10,20 +10,21 @@ import { createDatasetQualityESClient } from '../../../utils'; import { updateComponentTemplate } from './update_component_template'; import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; import { UpdateFieldLimitResponse } from '../../../../common/api_types'; +import { getDataStreamSettings } from '../get_data_stream_details'; export async function updateFieldLimit({ esClient, - indexTemplate, - lastBackingIndex, newFieldLimit, + dataStream, }: { esClient: ElasticsearchClient; - indexTemplate: string; - lastBackingIndex: string; newFieldLimit: number; + dataStream: string; }): Promise { const datasetQualityESClient = createDatasetQualityESClient(esClient); + const { lastBackingIndex, indexTemplate } = await getDataStreamSettings({ esClient, dataStream }); + const { acknowledged: isComponentTemplateUpdated, componentTemplateName, diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts index 151aabbbc059f3..45f37b44983aa1 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts @@ -104,6 +104,13 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); }); + it('returns only privileges if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApiAs('datasetQualityMonitorUser', nonExistentDataStream); + expect(resp.body).eql(defaultDataStreamPrivileges); + }); + it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts index 4a70cd1b2298c3..a132bc01c97207 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts @@ -30,6 +30,10 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { const serviceName = 'my-service'; const hostName = 'synth-host'; + const defaultDataStreamPrivileges = { + datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, + }; + async function callApi( dataStream: string, roleAuthc: RoleCredentials, @@ -87,6 +91,13 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); }); + it('returns only privileges if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApi(nonExistentDataStream, roleAuthc, internalReqHeader); + expect(resp.body).eql(defaultDataStreamPrivileges); + }); + it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, From 54c07c766154983868a6893e73ee33fa2f23d8e4 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 16 Oct 2024 11:57:07 +0200 Subject: [PATCH 15/28] Fix more review comments --- .../degraded_field_flyout/index.tsx | 8 +- .../field_limit_documentation_link.tsx | 1 - .../field_limit/message_callout.tsx | 90 ++++++++++--------- .../possible_mitigations/manual/index.tsx | 20 +++-- 4 files changed, 64 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 3af49c6327a21d..0bff62ab3820b3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -157,8 +157,12 @@ export default function DegradedFieldFlyout() { - - {isUserViewingTheIssueOnLatestBackingIndex && } + {isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + + )} ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx index 2b69f5373f252c..4a92367b2b1118 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -19,7 +19,6 @@ export function FieldLimitDocLink() { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx index 4993fb7a148f96..60093d68429195 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -6,8 +6,7 @@ */ import React, { useCallback } from 'react'; -import { EuiButton, EuiCallOut, EuiFlexItem } from '@elastic/eui'; -import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants'; +import { EuiButton, EuiCallOut } from '@elastic/eui'; import { fieldLimitMitigationFailedMessage, fieldLimitMitigationFailedMessageDescription, @@ -39,63 +38,68 @@ export function MessageCallout() { export function SuccessCallout() { const { - services: { application }, + services: { + application, + share: { + url: { locators }, + }, + }, } = useKibanaContextForPlugin(); const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); const { name } = datasetDetails; + const componentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }); + const onClickHandler = useCallback(async () => { - await application.navigateToApp(MANAGEMENT_APP_ID, { - path: `/data/index_management/component_templates/${getComponentTemplatePrefixFromIndexTemplate( - dataStreamSettings?.indexTemplate ?? name - )}@custom`, - openInNewTab: true, - }); - }, [application, dataStreamSettings?.indexTemplate, name]); + if (componentTemplateUrl) { + await application.navigateToUrl(componentTemplateUrl); + } + }, [application, componentTemplateUrl]); return ( - - + - - {fieldLimitMitigationSuccessComponentTemplateLinkText} - - - + {fieldLimitMitigationSuccessComponentTemplateLinkText} + + ); } export function ManualRolloverCallout() { const { triggerRollover } = useDegradedFields(); return ( - - +

{fieldLimitMitigationFailedMessageDescription}

+ -

{fieldLimitMitigationFailedMessageDescription}

- - {fieldLimitMitigationRolloverButton} - - - + {fieldLimitMitigationRolloverButton} + + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index 2b133ad8045b2b..fb6692a8191a76 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -40,15 +40,17 @@ function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); const { name } = datasetDetails; - const componentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl( - isIntegration - ? { - componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( - dataStreamSettings?.indexTemplate ?? name - )}@custom`, - } - : { indexTemplate: dataStreamSettings?.indexTemplate } - ); + const customComponentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ + componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( + dataStreamSettings?.indexTemplate ?? name + )}@custom`, + }); + + const indexTemplateUrl = locators + .get('MANAGEMENT_APP_LOCATOR') + ?.useUrl({ indexTemplate: dataStreamSettings?.indexTemplate }); + + const componentTemplateUrl = isIntegration ? customComponentTemplateUrl : indexTemplateUrl; return ( From 14f8dade0df2f5e602ec2669fa0488496ddbcd35 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 16 Oct 2024 12:28:26 +0200 Subject: [PATCH 16/28] Fix message callout and state machine --- .../field_limit/message_callout.tsx | 22 +++++-------------- .../state_machine.ts | 5 ++--- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx index 60093d68429195..bb96d47a525302 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -5,8 +5,8 @@ * 2.0. */ -import React, { useCallback } from 'react'; -import { EuiButton, EuiCallOut } from '@elastic/eui'; +import React from 'react'; +import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui'; import { fieldLimitMitigationFailedMessage, fieldLimitMitigationFailedMessageDescription, @@ -39,7 +39,6 @@ export function MessageCallout() { export function SuccessCallout() { const { services: { - application, share: { url: { locators }, }, @@ -54,29 +53,20 @@ export function SuccessCallout() { )}@custom`, }); - const onClickHandler = useCallback(async () => { - if (componentTemplateUrl) { - await application.navigateToUrl(componentTemplateUrl); - } - }, [application, componentTemplateUrl]); - return ( - {fieldLimitMitigationSuccessComponentTemplateLinkText} - + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index ed573fdddb8d85..ac3bcd6421925a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -377,9 +377,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, }, mitigations: { - initial: 'pending', states: { - pending: {}, savingNewFieldLimit: { invoke: { src: 'saveNewFieldLimit', @@ -668,7 +666,8 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ dataStream: context.showCurrentQualityIssues && 'dataStreamSettings' in context && - context.dataStreamSettings + context.dataStreamSettings && + context.dataStreamSettings.lastBackingIndexName ? context.dataStreamSettings.lastBackingIndexName : context.dataStream, start, From 8887f70d1bac35c85436a3977f4ef18ee1c2881c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 16 Oct 2024 15:59:22 +0200 Subject: [PATCH 17/28] Fix checking of response from settings API --- .../data_streams/update_field_limit/index.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts index d7bad9d7fcbe87..4e6a42248c17b1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -23,7 +23,19 @@ export async function updateFieldLimit({ }): Promise { const datasetQualityESClient = createDatasetQualityESClient(esClient); - const { lastBackingIndex, indexTemplate } = await getDataStreamSettings({ esClient, dataStream }); + const { lastBackingIndexName, indexTemplate } = await getDataStreamSettings({ + esClient, + dataStream, + }); + + if (!lastBackingIndexName || !indexTemplate) { + return { + isComponentTemplateUpdated: false, + isLatestBackingIndexUpdated: false, + customComponentTemplateName: '', + error: 'Data stream does not exists', + }; + } const { acknowledged: isComponentTemplateUpdated, @@ -43,7 +55,7 @@ export async function updateFieldLimit({ const { acknowledged: isLatestBackingIndexUpdated, error: errorUpdatingBackingIndex } = await updateLastBackingIndexSettings({ datasetQualityESClient, - lastBackingIndex, + lastBackingIndex: lastBackingIndexName, newFieldLimit, }); From cd3fa22f343eccbbb7a215a0ddb4c27373ce9186 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 17 Oct 2024 14:55:45 +0200 Subject: [PATCH 18/28] Migrate and rewrote Deployment Agnostic Tests for Settings --- .../get_data_stream_details/index.ts | 2 - .../dataset_quality/data_stream_settings.ts | 244 ++++++++++++++++++ .../dataset_quality/degraded_field_analyze.ts | 2 +- .../observability/dataset_quality/index.ts | 1 + .../dataset_quality/integrations.ts | 54 ++-- .../dataset_quality/{ => utils}/es_utils.ts | 18 ++ .../services/package_api.ts | 47 ++-- .../data_streams/data_stream_settings.spec.ts | 159 ------------ .../data_stream_settings.ts | 130 ---------- 9 files changed, 315 insertions(+), 342 deletions(-) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts rename x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/{ => utils}/es_utils.ts (63%) delete mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 79180801c6c0ef..288eff11b92a86 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -29,8 +29,6 @@ export async function getDataStreamSettings({ esClient: ElasticsearchClient; dataStream: string; }): Promise { - throwIfInvalidDataStreamParams(dataStream); - const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([ getDataStreamCreatedOn(esClient, dataStream), dataStreamService.getMatchingDataStreams(esClient, dataStream), diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts new file mode 100644 index 00000000000000..53975768cd88e3 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { + createBackingIndexNameWithoutVersion, + getDataStreamSettingsOfEarliestIndex, + rolloverDataStream, +} from './utils/es_utils'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const config = getService('config'); + const isServerless = !!config.get('serverless'); + const start = '2024-09-20T11:00:00.000Z'; + const end = '2024-09-20T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${type}-${syntheticsDataset}-${namespace}`; + + const defaultDataStreamPrivileges = { + datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, + }; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.get( + `/internal/dataset_quality/data_streams/${dataStream}/settings` + ); + } + + describe('Dataset quality settings', function () { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + }); + + it('returns only privileges if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: nonExistentDataStream, + }, + }); + expect(resp.body).eql(defaultDataStreamPrivileges); + }); + + describe('gets the data stream settings for non integrations', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly when available for non integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, dataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + dataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + + describe('gets the data stream settings for integrations', () => { + before(async () => { + await packageApi.installPackage({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + pkg: 'synthetics', + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + pkg: 'synthetics', + }); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly when available for non integration', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.indexTemplate).to.be('logs'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); + }); + + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, syntheticsDataStreamName); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + syntheticsDataStreamName + ); + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: syntheticsDataStreamName, + }, + }); + + if (!isServerless) { + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + } + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: syntheticsDataset, + namespace, + })}-000002` + ); + expect(resp.body.indexTemplate).to.be('logs'); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts index ed06b81d647fb9..e9e2665548764d 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts @@ -9,7 +9,7 @@ import expect from '@kbn/expect'; import { log, timerange } from '@kbn/apm-synthtrace-client'; import { SupertestWithRoleScopeType } from '../../../services'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; -import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './es_utils'; +import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './utils/es_utils'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; const MORE_THAN_1024_CHARS = diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0c660dda0a4455..ba98777e674b1d 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) describe('Dataset quality', () => { loadTestFile(require.resolve('./integrations')); loadTestFile(require.resolve('./degraded_field_analyze')); + loadTestFile(require.resolve('./data_stream_settings')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts index 910dd84bb309ee..d832a723959809 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts @@ -5,15 +5,14 @@ * 2.0. */ -import { RoleCredentials, InternalRequestHeader } from '@kbn/ftr-common-functional-services'; import expect from '@kbn/expect'; import { APIReturnType } from '@kbn/dataset-quality-plugin/common/rest'; import { CustomIntegration } from '../../../services/package_api'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { - const samlAuth = getService('samlAuth'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); const packageApi = getService('packageApi'); const endpoint = 'GET /internal/dataset_quality/integrations'; @@ -33,31 +32,28 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ]; async function callApiAs({ - roleAuthc, - headers, + roleScopedSupertestWithCookieCredentials, }: { - roleAuthc: RoleCredentials; - headers: InternalRequestHeader; + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; }): Promise { - const { body } = await supertestWithoutAuth - .get('/internal/dataset_quality/integrations') - .set(roleAuthc.apiKeyHeader) - .set(headers); + const { body } = await roleScopedSupertestWithCookieCredentials.get( + '/internal/dataset_quality/integrations' + ); return body; } describe('Integrations', () => { - let adminRoleAuthc: RoleCredentials; - let internalHeaders: InternalRequestHeader; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; before(async () => { - adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); - internalHeaders = samlAuth.getInternalRequestHeader(); - }); - - after(async () => { - await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); }); describe('gets the installed integrations', () => { @@ -65,7 +61,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await Promise.all( integrationPackages.map((pkg) => packageApi.installPackage({ - roleAuthc: adminRoleAuthc, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, pkg, }) ) @@ -74,8 +70,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('returns all installed integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ @@ -91,7 +86,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { async () => await Promise.all( integrationPackages.map((pkg) => - packageApi.uninstallPackage({ roleAuthc: adminRoleAuthc, pkg }) + packageApi.uninstallPackage({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + pkg, + }) ) ) ); @@ -101,15 +99,17 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { before(async () => { await Promise.all( customIntegrations.map((customIntegration: CustomIntegration) => - packageApi.installCustomIntegration({ roleAuthc: adminRoleAuthc, customIntegration }) + packageApi.installCustomIntegration({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + customIntegration, + }) ) ); }); it('returns custom integrations and its datasets map', async () => { const body = await callApiAs({ - roleAuthc: adminRoleAuthc, - headers: internalHeaders, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, }); expect(body.integrations.map((integration: Integration) => integration.name)).to.eql([ @@ -126,7 +126,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await Promise.all( customIntegrations.map((customIntegration: CustomIntegration) => packageApi.uninstallPackage({ - roleAuthc: adminRoleAuthc, + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, pkg: customIntegration.integrationName, }) ) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts similarity index 63% rename from x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts index 0e041781122cdc..a2fa712ba3be76 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/utils/es_utils.ts @@ -39,3 +39,21 @@ export async function setDataStreamSettings( settings, }); } + +export async function rolloverDataStream(es: Client, name: string) { + return es.indices.rollover({ alias: name }); +} + +export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: string) { + const matchingIndexesObj = await es.indices.getSettings({ index: name }); + + const matchingIndexes = Object.keys(matchingIndexesObj ?? {}); + matchingIndexes.sort((a, b) => { + return ( + Number(matchingIndexesObj[a].settings?.index?.creation_date) - + Number(matchingIndexesObj[b].settings?.index?.creation_date) + ); + }); + + return matchingIndexesObj[matchingIndexes[0]].settings; +} diff --git a/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts b/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts index 4406147f6b7564..c97b674e7b3a77 100644 --- a/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts +++ b/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts @@ -5,8 +5,7 @@ * 2.0. */ -import { RoleCredentials } from '@kbn/ftr-common-functional-services'; -import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '.'; export interface CustomIntegration { integrationName: string; @@ -18,50 +17,52 @@ export interface IntegrationDataset { type: 'logs' | 'metrics' | 'synthetics' | 'traces'; } -export function PackageApiProvider({ getService }: DeploymentAgnosticFtrProviderContext) { - const samlAuth = getService('samlAuth'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); - +export function PackageApiProvider() { return { async installCustomIntegration({ - roleAuthc, + roleScopedSupertestWithCookieCredentials, customIntegration, }: { - roleAuthc: RoleCredentials; + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; customIntegration: CustomIntegration; }) { const { integrationName, datasets } = customIntegration; - const { body } = await supertestWithoutAuth + const { body } = await roleScopedSupertestWithCookieCredentials .post(`/api/fleet/epm/custom_integrations`) - .set(roleAuthc.apiKeyHeader) - .set(samlAuth.getInternalRequestHeader()) .send({ integrationName, datasets }); return body; }, - async installPackage({ roleAuthc, pkg }: { roleAuthc: RoleCredentials; pkg: string }) { + async installPackage({ + roleScopedSupertestWithCookieCredentials, + pkg, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + pkg: string; + }) { const { body: { item: { latestVersion: version }, }, - } = await supertestWithoutAuth + } = await roleScopedSupertestWithCookieCredentials .get(`/api/fleet/epm/packages/${pkg}`) - .set(roleAuthc.apiKeyHeader) - .set(samlAuth.getInternalRequestHeader()) .send({ force: true }); - const { body } = await supertestWithoutAuth + const { body } = await roleScopedSupertestWithCookieCredentials .post(`/api/fleet/epm/packages/${pkg}/${version}`) - .set(roleAuthc.apiKeyHeader) - .set(samlAuth.getInternalRequestHeader()) .send({ force: true }); return body; }, - async uninstallPackage({ roleAuthc, pkg }: { roleAuthc: RoleCredentials; pkg: string }) { - const { body } = await supertestWithoutAuth - .delete(`/api/fleet/epm/packages/${pkg}`) - .set(roleAuthc.apiKeyHeader) - .set(samlAuth.getInternalRequestHeader()); + async uninstallPackage({ + roleScopedSupertestWithCookieCredentials, + pkg, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + pkg: string; + }) { + const { body } = await roleScopedSupertestWithCookieCredentials.delete( + `/api/fleet/epm/packages/${pkg}` + ); return body; }, }; diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts deleted file mode 100644 index 45f37b44983aa1..00000000000000 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import { DatasetQualityApiClientKey } from '../../common/config'; -import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { - expectToReject, - getDataStreamSettingsOfEarliestIndex, - rolloverDataStream, -} from '../../utils'; -import { createBackingIndexNameWithoutVersion } from './es_utils'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const synthtrace = getService('logSynthtraceEsClient'); - const esClient = getService('es'); - const datasetQualityApiClient = getService('datasetQualityApiClient'); - const pkgService = getService('packageService'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'synth.1'; - const integrationDataset = 'apache.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - const pkg = { - name: 'apache', - version: '1.14.0', - }; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { - return await datasetQualityApiClient[user]({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - }); - } - - registry.when('DataStream Settings', { config: 'basic' }, () => { - describe('gets the data stream settings', () => { - before(async () => { - // Install Integration and ingest logs for it - await pkgService.installPackage(pkg); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(integrationDataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - // Ingest basic logs - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApiAs('datasetQualityMonitorUser', encodeURIComponent(' ')) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApiAs('datasetQualityMonitorUser', nonExistentDataStream); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${integrationDataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${integrationDataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.integration).to.be('apache'); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset: integrationDataset, - namespace, - })}-000001` - ); - expect(resp.body.datasetUserPrivileges).to.eql( - defaultDataStreamPrivileges.datasetUserPrivileges - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${dataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - - after(async () => { - await synthtrace.clean(); - await pkgService.uninstallPackage(pkg); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts deleted file mode 100644 index a132bc01c97207..00000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { log, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; -import type { InternalRequestHeader, RoleCredentials } from '../../../../shared/services'; -import { expectToReject, getDataStreamSettingsOfEarliestIndex, rolloverDataStream } from './utils'; -import { - DatasetQualityApiClient, - DatasetQualityApiError, -} from './common/dataset_quality_api_supertest'; -import { DatasetQualityFtrContextProvider } from './common/services'; -import { createBackingIndexNameWithoutVersion } from './utils'; - -export default function ({ getService }: DatasetQualityFtrContextProvider) { - const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); - const synthtrace = getService('logSynthtraceEsClient'); - const svlCommonApi = getService('svlCommonApi'); - const svlUserManager = getService('svlUserManager'); - const esClient = getService('es'); - const start = '2023-12-11T18:00:00.000Z'; - const end = '2023-12-11T18:01:00.000Z'; - const type = 'logs'; - const dataset = 'nginx.access'; - const namespace = 'default'; - const serviceName = 'my-service'; - const hostName = 'synth-host'; - - const defaultDataStreamPrivileges = { - datasetUserPrivileges: { canRead: true, canMonitor: true, canViewIntegrations: true }, - }; - - async function callApi( - dataStream: string, - roleAuthc: RoleCredentials, - internalReqHeader: InternalRequestHeader - ) { - return await datasetQualityApiClient.slsUser({ - endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', - params: { - path: { - dataStream, - }, - }, - roleAuthc, - internalReqHeader, - }); - } - - describe('gets the data stream settings', () => { - let roleAuthc: RoleCredentials; - let internalReqHeader: InternalRequestHeader; - before(async () => { - roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin'); - internalReqHeader = svlCommonApi.getInternalRequestHeader(); - await synthtrace.index([ - timerange(start, end) - .interval('1m') - .rate(1) - .generator((timestamp) => - log - .create() - .message('This is a log message') - .timestamp(timestamp) - .dataset(dataset) - .namespace(namespace) - .defaults({ - 'log.file.path': '/my-service.log', - 'service.name': serviceName, - 'host.name': hostName, - }) - ), - ]); - }); - - after(async () => { - await synthtrace.clean(); - await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc); - }); - - it('returns error when dataStream param is not provided', async () => { - const expectedMessage = 'Data Stream name cannot be empty'; - const err = await expectToReject(() => - callApi(encodeURIComponent(' '), roleAuthc, internalReqHeader) - ); - expect(err.res.status).to.be(400); - expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); - }); - - it('returns only privileges if matching data stream is not available', async () => { - const nonExistentDataSet = 'Non-existent'; - const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; - const resp = await callApi(nonExistentDataStream, roleAuthc, internalReqHeader); - expect(resp.body).eql(defaultDataStreamPrivileges); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ - type, - dataset, - namespace, - })}-000001` - ); - }); - - it('returns "createdOn" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { - await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${dataset}-${namespace}` - ); - const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.lastBackingIndexName).to.be( - `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` - ); - }); - }); -} From 413373d5ba16aef5256f7f772b867e03ad729f23 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 17 Oct 2024 15:24:42 +0200 Subject: [PATCH 19/28] Add Deployment Agnostic Tests for rollover API --- .../dataset_quality/data_stream_rollover.ts | 86 +++++++++++++++++++ .../observability/dataset_quality/index.ts | 1 + 2 files changed, 87 insertions(+) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts new file mode 100644 index 00000000000000..d11d93d9eae510 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_rollover.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.post( + `/internal/dataset_quality/data_streams/${dataStream}/rollover` + ); + } + + describe('Datastream Rollover', function () { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('returns acknowledged when rollover is successful', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + expect(resp.body.acknowledged).to.be(true); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index ba98777e674b1d..dd98fc7d70da21 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -12,5 +12,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./integrations')); loadTestFile(require.resolve('./degraded_field_analyze')); loadTestFile(require.resolve('./data_stream_settings')); + loadTestFile(require.resolve('./data_stream_rollover')); }); } From e76d91e1ae6f459704eeeca11a0a53f2cc1df0ce Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 17 Oct 2024 17:05:50 +0200 Subject: [PATCH 20/28] Add Deployment Agnostic Tests for Update field Limit API --- .../dataset_quality/data_stream_settings.ts | 4 +- .../observability/dataset_quality/index.ts | 1 + .../dataset_quality/update_field_limit.ts | 173 ++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts index 53975768cd88e3..d75b8940987cdf 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -157,7 +157,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { before(async () => { await packageApi.installPackage({ roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, - pkg: 'synthetics', + pkg: syntheticsDataset, }); await synthtrace.index([ timerange(start, end) @@ -182,7 +182,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await synthtrace.clean(); await packageApi.uninstallPackage({ roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, - pkg: 'synthetics', + pkg: syntheticsDataset, }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index dd98fc7d70da21..7e555b7a310e1a 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -13,5 +13,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./degraded_field_analyze')); loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./data_stream_rollover')); + loadTestFile(require.resolve('./update_field_limit')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts new file mode 100644 index 00000000000000..5cb0558b8069ea --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { log, timerange } from '@kbn/apm-synthtrace-client'; + +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { SupertestWithRoleScopeType } from '../../../services'; +import { createBackingIndexNameWithoutVersion, rolloverDataStream } from './utils/es_utils'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const packageApi = getService('packageApi'); + const start = '2024-10-17T11:00:00.000Z'; + const end = '2024-10-17T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const invalidDataset = 'invalid'; + const integrationsDataset = 'nginx.access'; + const pkg = 'nginx'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const invalidDataStreamName = `${type}-${invalidDataset}-${namespace}`; + const integrationsDataStreamName = `${type}-${integrationsDataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream, fieldLimit }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + fieldLimit: number; + }; + }) { + return roleScopedSupertestWithCookieCredentials + .put(`/internal/dataset_quality/data_streams/${dataStream}/update_field_limit`) + .send({ + newFieldLimit: fieldLimit, + }); + } + + describe('Update field limit', function () { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + await packageApi.installPackage({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + pkg, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(integrationsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await packageApi.uninstallPackage({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + pkg, + }); + }); + + it('should handles failure gracefully when invalid datastream provided ', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: invalidDataStreamName, + fieldLimit: 10, + }, + }); + + expect(resp.body.isComponentTemplateUpdated).to.be(false); + expect(resp.body.isLatestBackingIndexUpdated).to.be(false); + expect(resp.body.customComponentTemplateName).to.be(''); + expect(resp.body.error).to.be('Data stream does not exists'); + }); + + it('should update last backing index and custom component template', async () => { + // We rollover the data stream to create a new backing index + await rolloverDataStream(esClient, integrationsDataStreamName); + + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: integrationsDataStreamName, + fieldLimit: 50, + }, + }); + + expect(resp.body.isComponentTemplateUpdated).to.be(true); + expect(resp.body.isLatestBackingIndexUpdated).to.be(true); + expect(resp.body.customComponentTemplateName).to.be(`${type}-${integrationsDataset}@custom`); + expect(resp.body.error).to.be(undefined); + + const { component_templates: componentTemplates } = + await esClient.cluster.getComponentTemplate({ + name: `${type}-${integrationsDataset}@custom`, + }); + + const customTemplate = componentTemplates.filter( + (tmp) => tmp.name === `${type}-${integrationsDataset}@custom` + ); + + expect(customTemplate).to.have.length(1); + expect( + customTemplate[0].component_template.template.settings.index.mapping.total_fields.limit + ).to.be('50'); + + const settingsForAllIndices = await esClient.indices.getSettings({ + index: integrationsDataStreamName, + }); + + const backingIndexWithoutVersion = createBackingIndexNameWithoutVersion({ + type, + dataset: integrationsDataset, + namespace, + }); + const settingsForLastBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000002'].settings; + const settingsForPreviousBackingIndex = + settingsForAllIndices[backingIndexWithoutVersion + '-000001'].settings; + + // Only the Last Backing Index should have the updated limit and not the one previous to it + expect(settingsForLastBackingIndex.index.mapping.total_fields.limit).to.be('50'); + + // The previous one should have the default limit of 1000 + expect(settingsForPreviousBackingIndex.index.mapping.total_fields.limit).to.be('1000'); + + // Rollover to test custom component template + await rolloverDataStream(esClient, integrationsDataStreamName); + + const settingsForLatestBackingIndex = await esClient.indices.getSettings({ + index: backingIndexWithoutVersion + '-000003', + }); + + // The new backing index should read settings from custom component template + expect( + settingsForLatestBackingIndex[backingIndexWithoutVersion + '-000003'].settings.index.mapping + .total_fields.limit + ).to.be('50'); + }); + }); +} From a9baa8303dce9a3ffbe6b8220fe46cd4ac381ea7 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 17 Oct 2024 21:18:15 +0200 Subject: [PATCH 21/28] Fix checktypes and lint issues --- .../dataset_quality/update_field_limit.ts | 11 +++++------ .../dataset_quality_api_integration/index.ts | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts index 5cb0558b8069ea..8f4b58cc758e68 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts @@ -20,7 +20,6 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const start = '2024-10-17T11:00:00.000Z'; const end = '2024-10-17T11:01:00.000Z'; const type = 'logs'; - const dataset = 'synth'; const invalidDataset = 'invalid'; const integrationsDataset = 'nginx.access'; const pkg = 'nginx'; @@ -133,7 +132,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { expect(customTemplate).to.have.length(1); expect( - customTemplate[0].component_template.template.settings.index.mapping.total_fields.limit + customTemplate[0].component_template.template.settings?.index?.mapping?.total_fields?.limit ).to.be('50'); const settingsForAllIndices = await esClient.indices.getSettings({ @@ -151,10 +150,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { settingsForAllIndices[backingIndexWithoutVersion + '-000001'].settings; // Only the Last Backing Index should have the updated limit and not the one previous to it - expect(settingsForLastBackingIndex.index.mapping.total_fields.limit).to.be('50'); + expect(settingsForLastBackingIndex?.index?.mapping?.total_fields?.limit).to.be('50'); // The previous one should have the default limit of 1000 - expect(settingsForPreviousBackingIndex.index.mapping.total_fields.limit).to.be('1000'); + expect(settingsForPreviousBackingIndex?.index?.mapping?.total_fields?.limit).to.be('1000'); // Rollover to test custom component template await rolloverDataStream(esClient, integrationsDataStreamName); @@ -165,8 +164,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { // The new backing index should read settings from custom component template expect( - settingsForLatestBackingIndex[backingIndexWithoutVersion + '-000003'].settings.index.mapping - .total_fields.limit + settingsForLatestBackingIndex[backingIndexWithoutVersion + '-000003'].settings?.index + ?.mapping?.total_fields?.limit ).to.be('50'); }); }); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts index 86c85271398464..39b6a3cb476c1d 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts @@ -9,7 +9,6 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Dataset Quality', function () { loadTestFile(require.resolve('./data_stream_details')); - loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./degraded_field_values')); }); } From 73662db0ecb482a09e3bd34431073da11e603f0f Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 18 Oct 2024 15:22:25 +0200 Subject: [PATCH 22/28] Fix ingest pipeline design --- .../dataset_quality/common/translations.ts | 13 +- .../field_limit_documentation_link.tsx | 2 +- .../manual/component_template_link.tsx | 65 +++++++++ .../possible_mitigations/manual/index.tsx | 101 +------------- .../manual/pipeline_link.tsx | 124 ++++++++++++++++++ 5 files changed, 205 insertions(+), 100 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 0a6596a868f775..397b77541808eb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -645,9 +645,16 @@ export const fieldLimitMitigationRolloverButton = i18n.translate( } ); -export const manualMitigationCustomPipelineText = i18n.translate( - 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomPipelineText', +export const manualMitigationCustomPipelineCopyPipelineNameAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.copyPipelineNameAriaText', { - defaultMessage: 'Copy the name below to add or edit the custom pipeline', + defaultMessage: 'Copy pipeline name', + } +); + +export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.createEditPipelineLink', + { + defaultMessage: 'create or edit the pipeline', } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx index 4a92367b2b1118..3da690c1990a81 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -17,7 +17,7 @@ export function FieldLimitDocLink() { return ( { + await application.navigateToApp(MANAGEMENT_APP_ID, { + path: componentTemplateUrl, + openInNewTab: true, + }); + }, [application, componentTemplateUrl]); + + return ( + + + + + +

{otherMitigationsCustomComponentTemplate}

+ + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index fb6692a8191a76..4e8eaf5a6c8a92 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -6,110 +6,19 @@ */ import React from 'react'; -import { EuiLink, EuiPanel, EuiSpacer, EuiText, EuiCopy, EuiButtonEmpty } from '@elastic/eui'; -import { useKibanaContextForPlugin } from '../../../../../utils'; -import { - manualMitigationCustomPipelineText, - otherMitigationsCustomComponentTemplate, - otherMitigationsCustomIngestPipeline, -} from '../../../../../../common/translations'; +import { EuiSpacer } from '@elastic/eui'; import { useDatasetQualityDetailsState } from '../../../../../hooks'; -import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; +import { CreateEditComponentTemplateLink } from './component_template_link'; +import { CreateEditPipelineLink } from './pipeline_link'; export function ManualMitigations() { const { integrationDetails } = useDatasetQualityDetailsState(); const isIntegration = !!integrationDetails?.integration; return ( <> - + - + ); } - -function EditComponentTemplate({ isIntegration }: { isIntegration: boolean }) { - const { - services: { - share: { - url: { locators }, - }, - }, - } = useKibanaContextForPlugin(); - - const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); - const { name } = datasetDetails; - - const customComponentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ - componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( - dataStreamSettings?.indexTemplate ?? name - )}@custom`, - }); - - const indexTemplateUrl = locators - .get('MANAGEMENT_APP_LOCATOR') - ?.useUrl({ indexTemplate: dataStreamSettings?.indexTemplate }); - - const componentTemplateUrl = isIntegration ? customComponentTemplateUrl : indexTemplateUrl; - - return ( - - - {otherMitigationsCustomComponentTemplate} - - - ); -} - -function EditPipeline({ isIntegration }: { isIntegration: boolean }) { - const { - services: { - share: { - url: { locators }, - }, - }, - } = useKibanaContextForPlugin(); - - const { datasetDetails } = useDatasetQualityDetailsState(); - const { type, name } = datasetDetails; - - const copyText = isIntegration ? `${type}-${name}@custom` : `${type}@custom`; - - const pipelineUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ pipeline: '' }); - - return ( - - - {otherMitigationsCustomIngestPipeline} - - - -

{manualMitigationCustomPipelineText}

- - {(copy) => ( - - {copyText} - - )} - - - - ); -} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx new file mode 100644 index 00000000000000..28ec9d2dab9a58 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { + copyToClipboard, + EuiAccordion, + EuiButtonIcon, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiLink, + EuiPanel, + EuiSpacer, + EuiTitle, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { + manualMitigationCustomPipelineCopyPipelineNameAriaText, + manualMitigationCustomPipelineCreateEditPipelineLink, + otherMitigationsCustomIngestPipeline, +} from '../../../../../../common/translations'; +import { useKibanaContextForPlugin } from '../../../../../utils'; +import { useDatasetQualityDetailsState } from '../../../../../hooks'; + +export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boolean }) { + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const accordionId = useGeneratedHtmlId({ + prefix: otherMitigationsCustomIngestPipeline, + }); + + const accordionTitle = ( + +
{otherMitigationsCustomIngestPipeline}
+ + ); + + const { datasetDetails } = useDatasetQualityDetailsState(); + const { type, name } = datasetDetails; + + const copyText = isIntegration ? `${type}-${name}@custom` : `${type}@custom`; + + const pipelineUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ pipeline: ' ' }); + + return ( + + + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '1.', + })} + + ), + }} + /> + + copyToClipboard(copyText)} + /> + } + readOnly={true} + aria-label={manualMitigationCustomPipelineCopyPipelineNameAriaText} + value={copyText} + data-test-subj="datasetQualityManualMitigationsPipelineName" + fullWidth + /> + + + {i18n.translate('xpack.datasetQuality.editPipeline.strong.Label', { + defaultMessage: '2.', + })} + + ), + createEditPipelineLink: ( + + {manualMitigationCustomPipelineCreateEditPipelineLink} + + ), + }} + /> + + + + ); +} From d04fac54c733432f575988793207beab76f0364e Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Mon, 21 Oct 2024 13:58:02 +0200 Subject: [PATCH 23/28] Add stateful ftr tests --- .../field_limit_documentation_link.tsx | 2 +- .../field_limit/field_mapping_limit.tsx | 18 - .../increase_field_mapping_limit.tsx | 9 +- .../field_limit/message_callout.tsx | 3 +- .../manual/component_template_link.tsx | 20 +- .../manual/pipeline_link.tsx | 11 +- .../possible_mitigations/title.tsx | 5 +- .../state_machine.ts | 2 + .../dataset_quality/degraded_field_flyout.ts | 673 ++++++++++++++---- .../page_objects/dataset_quality.ts | 21 + 10 files changed, 597 insertions(+), 167 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx index 3da690c1990a81..0dd80bb120e543 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -17,7 +17,7 @@ export function FieldLimitDocLink() { return ( - {degradedFieldAnalysis?.isFieldLimitIssue && ( - - - - {degradedFieldCurrentFieldLimitColumnName} - - - - {degradedFieldAnalysis.totalFieldLimit} - - - )} -

{fieldLimitMitigationConsiderationText}

diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx index 5ab701776be5a0..e6cf1e1899068e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx @@ -40,11 +40,14 @@ export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit }; return ( - + @@ -53,7 +56,7 @@ export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit validateNewLimit(e.target.value)} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx index bb96d47a525302..8a1e85941ebd18 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -58,9 +58,10 @@ export function SuccessCallout() { title={fieldLimitMitigationSuccessMessage} color="success" iconType="checkInCircleFilled" + data-test-subj="datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout" > (null); + const { dataStreamSettings, datasetDetails } = useDatasetQualityDetailsState(); const { name } = datasetDetails; + const managementAppLocator = locators.get('MANAGEMENT_APP_LOCATOR'); + + useEffect(() => { + managementAppLocator + ?.getLocation({ indexTemplate: dataStreamSettings?.indexTemplate }) + .then(({ path }) => setTemplatePath(path)); + }, [locators, setTemplatePath, dataStreamSettings?.indexTemplate, managementAppLocator]); - const customComponentTemplateUrl = locators.get('MANAGEMENT_APP_LOCATOR')?.useUrl({ + const customComponentTemplateUrl = managementAppLocator?.useUrl({ componentTemplate: `${getComponentTemplatePrefixFromIndexTemplate( dataStreamSettings?.indexTemplate ?? name )}@custom`, }); - const indexTemplateUrl = locators - .get('MANAGEMENT_APP_LOCATOR') - ?.useUrl({ indexTemplate: dataStreamSettings?.indexTemplate }); - - const componentTemplateUrl = isIntegration ? customComponentTemplateUrl : indexTemplateUrl; + const componentTemplateUrl = isIntegration ? customComponentTemplateUrl : templatePath; const onClickHandler = useCallback(async () => { await application.navigateToApp(MANAGEMENT_APP_ID, { @@ -49,6 +54,7 @@ export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegrati @@ -85,12 +87,12 @@ export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boole copyToClipboard(copyText)} + onClick={() => copyToClipboard(pipelineName)} /> } readOnly={true} aria-label={manualMitigationCustomPipelineCopyPipelineNameAriaText} - value={copyText} + value={pipelineName} data-test-subj="datasetQualityManualMitigationsPipelineName" fullWidth /> @@ -109,6 +111,7 @@ export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boole createEditPipelineLink: ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx index 814f003feedd88..93e253b0f849c2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx @@ -17,7 +17,10 @@ export function PossibleMitigationTitle() { return ( - +

{possibleMitigationTitle}

 { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -90,32 +97,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - - describe('testing root cause for ignored fields', () => { + describe('detecting root cause for ignored fields', () => { before(async () => { + await synthtrace.clean(); // Create custom component template await synthtrace.createComponentTemplate( customComponentTemplateName, @@ -142,8 +126,12 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields in degraded DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -161,7 +149,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 42 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -176,8 +187,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 42, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -196,7 +212,31 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -205,9 +245,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 43 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 43, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -223,11 +284,34 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -235,8 +319,128 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ]); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -253,25 +457,161 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${nonIntegrationCustomName}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/app/management/data/index_management/component_templates/${integrationSpecificCustomName}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${integrationSpecificCustomName}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -289,7 +629,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -319,122 +659,191 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); - - expect(currentIssuesToggleState).to.be(false); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - - expect(rows.length).to.eql(3); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(newCurrentIssuesToggleState).to.be(true); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + const linkURL = await linkButton.getAttribute('href'); - expect(newRows.length).to.eql(2); + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + expandedDegradedField: 'cloud.project', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(true); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); - }); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + expect(currentFieldLimit).to.be(43); + expect(currentFieldLimitDisabledStatus).to.be('true'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); - }); + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue, 10); + const newFieldLimitDisabledStatus = await newFieldLimitInput.getAttribute('disabled'); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + // Should not allow values less than current limit of 43 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + // Should point to the component template page + expect( + ctLinkURL.endsWith( + `/app/management/data/index_management/component_templates/${type}-${nginxAccessDatasetName}@custom` + ) + ).to.be(true); + }); + + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); + + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index ccd48e220064a2..af1f6c94941ffd 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -239,6 +239,18 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, + generateBackingIndexNameWithoutVersion({ + type = 'logs', + dataset, + namespace = 'default', + }: { + type: string; + dataset: string; + namespace: string; + }) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; + }, + getDatasetsTable(): Promise { return testSubjects.find(testSubjectSelectors.datasetQualityTable); }, @@ -554,3 +566,12 @@ async function getDatasetTableHeaderTexts(tableWrapper: WebElementWrapper) { headerElementWrappers.map((headerElementWrapper) => headerElementWrapper.getVisibleText()) ); } + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} From 2c548f8ad8e8ade4482ca550ef9688078cd3f87c Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:50:09 +0000 Subject: [PATCH 24/28] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../possible_mitigations/field_limit/field_mapping_limit.tsx | 3 --- .../possible_mitigations/manual/pipeline_link.tsx | 2 -- 2 files changed, 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx index 4402f59b0f64b6..1056713ac20708 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -8,8 +8,6 @@ import React from 'react'; import { EuiAccordion, - EuiFlexGroup, - EuiFlexItem, EuiHorizontalRule, EuiPanel, EuiSpacer, @@ -18,7 +16,6 @@ import { useGeneratedHtmlId, } from '@elastic/eui'; import { - degradedFieldCurrentFieldLimitColumnName, fieldLimitMitigationConsiderationText, fieldLimitMitigationConsiderationText1, fieldLimitMitigationConsiderationText2, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx index da70ee22635fde..fcc781c58738de 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx @@ -13,8 +13,6 @@ import { EuiAccordion, EuiButtonIcon, EuiFieldText, - EuiFlexGroup, - EuiFlexItem, EuiHorizontalRule, EuiLink, EuiPanel, From 14dee9f4c6a22066e27249d4a87482f638cf2be8 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Mon, 21 Oct 2024 16:08:45 +0200 Subject: [PATCH 25/28] Fix tests for Logs DB and MKI issue --- .../custom_integration_mappings.ts | 177 ++++++++++++++++++ .../dataset_quality/degraded_field_flyout.ts | 11 +- 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts diff --git a/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 00000000000000..ada87a45723a23 --- /dev/null +++ b/x-pack/test/functional/apps/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index 79a97538269d63..34bf04be8b7431 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { MORE_THAN_1024_CHARS, } from './data'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { const PageObjects = getPageObjects([ @@ -39,6 +40,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; const nginxAccessDatasetName = 'nginx.access'; const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; @@ -99,7 +101,6 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid describe('detecting root cause for ignored fields', () => { before(async () => { - await synthtrace.clean(); // Create custom component template await synthtrace.createComponentTemplate( customComponentTemplateName, @@ -130,6 +131,12 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Install Nginx Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ // Ingest Degraded Logs with 25 fields in degraded DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) @@ -854,6 +861,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); }); From ab3855ee9be7aca1f672f20125648a75ab384f15 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 22 Oct 2024 15:09:51 +0200 Subject: [PATCH 26/28] Add serverless FTR tests --- .../dataset_quality/common/translations.ts | 7 + .../possible_mitigations/manual/index.tsx | 23 +- .../dataset_quality/degraded_field_flyout.ts | 2 +- .../page_objects/dataset_quality.ts | 4 + .../custom_integration_mappings.ts | 349 +++++++++ .../dataset_quality/degraded_field_flyout.ts | 685 ++++++++++++++---- 6 files changed, 933 insertions(+), 137 deletions(-) create mode 100644 x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 397b77541808eb..70c249e4dc9311 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -588,6 +588,13 @@ export const fieldLimitMitigationApplyButtonText = i18n.translate( } ); +export const otherMitigationsLoadingAriaText = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsLoadingText', + { + defaultMessage: 'Loading possible mitigations', + } +); + export const otherMitigationsCustomComponentTemplate = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.otherMitigationsCustomComponentTemplate', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index 4e8eaf5a6c8a92..f931f3461fb57f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -6,19 +6,34 @@ */ import React from 'react'; -import { EuiSpacer } from '@elastic/eui'; +import { EuiSkeletonRectangle, EuiSpacer } from '@elastic/eui'; import { useDatasetQualityDetailsState } from '../../../../../hooks'; import { CreateEditComponentTemplateLink } from './component_template_link'; import { CreateEditPipelineLink } from './pipeline_link'; +import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations'; export function ManualMitigations() { - const { integrationDetails } = useDatasetQualityDetailsState(); + const { integrationDetails, loadingState, dataStreamSettings } = useDatasetQualityDetailsState(); + const isIntegrationPresentInSettings = dataStreamSettings?.integration; const isIntegration = !!integrationDetails?.integration; + const { dataStreamSettingsLoading, integrationDetailsLoadings } = loadingState; + + const hasIntegrationCheckCompleted = + !dataStreamSettingsLoading && + ((isIntegrationPresentInSettings && !integrationDetailsLoadings) || + !isIntegrationPresentInSettings); + return ( - <> + - + ); } diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index 34bf04be8b7431..2df159b46ebb8b 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -40,9 +40,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; const nginxPkg = { name: 'nginx', diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index af1f6c94941ffd..0eab13cb8e11bb 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -204,6 +204,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv } }, + async waitUntilPossibleMitigationsLoaded() { + await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiSkeletonRectangle', 20 * 1000); + }, + async waitUntilDegradedFieldFlyoutLoaded() { await testSubjects.existOrFail(testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout); }, diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts new file mode 100644 index 00000000000000..214b68c1a25797 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/custom_mappings/custom_integration_mappings.ts @@ -0,0 +1,349 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const logsNginxMappings = (dataset: string): MappingTypeMapping => ({ + properties: { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, +}); + +const properties = { + '@timestamp': { + type: 'date', + ignore_malformed: false, + }, + cloud: { + properties: { + image: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + data_stream: { + properties: { + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + namespace: { + type: 'constant_keyword', + value: 'default', + }, + type: { + type: 'constant_keyword', + value: 'logs', + }, + }, + }, + ecs: { + properties: { + version: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + error: { + properties: { + message: { + type: 'match_only_text', + }, + }, + }, + event: { + properties: { + agent_id_status: { + type: 'keyword', + ignore_above: 1024, + }, + dataset: { + type: 'constant_keyword', + value: 'nginx.access', + }, + ingested: { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, + }, + module: { + type: 'constant_keyword', + value: 'nginx', + }, + original: { + type: 'keyword', + index: false, + doc_values: false, + }, + }, + }, + host: { + properties: { + containerized: { + type: 'boolean', + }, + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + os: { + properties: { + build: { + type: 'keyword', + ignore_above: 1024, + }, + codename: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + input: { + properties: { + type: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + log: { + properties: { + level: { + type: 'keyword', + ignore_above: 1024, + }, + offset: { + type: 'long', + }, + }, + }, + network: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + nginx: { + properties: { + access: { + properties: { + remote_ip_list: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, + }, + }, + }, + test_field: { + type: 'keyword', + ignore_above: 1024, + }, + tls: { + properties: { + established: { + type: 'boolean', + }, + }, + }, + trace: { + properties: { + id: { + type: 'keyword', + ignore_above: 1024, + }, + }, + }, +}; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 4072dcec8a25c3..1bf2d37433855c 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -17,6 +17,7 @@ import { } from './data'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; +import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects([ @@ -31,35 +32,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esClient = getService('es'); const retry = getService('retry'); const to = new Date().toISOString(); + const type = 'logs'; const degradedDatasetName = 'synth.degraded'; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const degradedDataStreamName = `${type}-${degradedDatasetName}-${defaultNamespace}`; const degradedDatasetWithLimitsName = 'synth.degraded.rca'; - const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const degradedDatasetWithLimitDataStreamName = `${type}-${degradedDatasetWithLimitsName}-${defaultNamespace}`; const serviceName = 'test_service'; const count = 5; const customComponentTemplateName = 'logs-synth@mappings'; - describe('Degraded fields flyout', function () { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - await PageObjects.svlCommonPage.loginWithPrivilegedRole(); - }); - - after(async () => { - await synthtrace.clean(); - }); + const nginxAccessDatasetName = 'nginx.access'; + const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; + const nginxPkg = { + name: 'nginx', + version: '1.23.0', + }; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.svlCommonPage.loginAsAdmin(); + }); + + after(async () => { + await synthtrace.clean(); + }); it('should open and close the flyout when user clicks on the expand button', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, @@ -88,30 +97,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - describe('testing root cause for ignored fields', () => { before(async () => { // Create custom component template @@ -140,8 +125,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { hidden: false, }, }); - // Ingest Degraded Logs with 25 fields + + // Install Nginx Integration and ingest logs for it + await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); + + // Create custom component template to avoid issues with LogsDB + await synthtrace.createComponentTemplate( + customComponentTemplateNameNginx, + logsNginxMappings(nginxAccessDatasetName) + ); + await synthtrace.index([ + // Ingest Degraded Logs with 25 fields timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -159,7 +154,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), - test_field: [MORE_THAN_1024_CHARS, 'hello world'], + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], }) .timestamp(timestamp) ); @@ -174,8 +192,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Ingest Degraded Logs with 26 field + // Set Limit of 42 + await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { + 'mapping.total_fields.limit': 43, + }); + await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -194,7 +217,31 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 44 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); @@ -203,9 +250,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: degradedDatasetWithLimitsName, + }) + '-000002', + { + 'mapping.total_fields.limit': 26, + } + ); + + // Set Limit of 44 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: nginxAccessDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 44, + } + ); - // Ingest docs with 26 fields again await synthtrace.index([ + // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -221,20 +289,164 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .service(serviceName) .namespace(defaultNamespace) .defaults({ - 'log.file.path': '/my-service.log', 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, 'hello world'], - 'cloud.region': 'us-east-1', + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), + // Ingest Degraded Logs with 43 fields in Nginx DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(nginxAccessDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), }) .timestamp(timestamp) ); }), ]); + await PageObjects.svlCommonPage.loginAsAdmin(); }); - describe('field character limit exceeded', () => { - it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(4); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(3); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + describe('character limit exceeded', () => { + it('should display cause as "field character limit exceeded" when a field is ignored due to character limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', @@ -251,25 +463,163 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display values when cause is "field ignored"', async () => { + it('should display values when cause is "field character limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'test_field', }); await retry.tryForTime(5000, async () => { - const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + const testFieldValue1Exists = await PageObjects.datasetQuality.doesTextExist( 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', MORE_THAN_1024_CHARS ); - expect(testFieldValueExists).to.be(true); + const testFieldValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + expect(testFieldValue1Exists).to.be(true); + expect(testFieldValue2Exists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display the maximum character limit when cause is "field character limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const limitValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit', + '1024' + ); + expect(limitValueExists).to.be(true); }); await PageObjects.datasetQuality.closeFlyout(); }); + + it('should show possible mitigation section with manual options for non integrations', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + // Should point to index template with the datastream name as value + expect(componentTemplateUrl).to.be( + `/data/index_management/templates/${degradedDatasetWithLimitDataStreamName}` + ); + + const nonIntegrationCustomName = `${type}@custom`; + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(nonIntegrationCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${nonIntegrationCustomName}` + ); + }); + + it('should show possible mitigation section with different manual options for integrations', async () => { + // Navigate to Integration Dataset + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'test_field', + }); + + await PageObjects.datasetQuality.waitUntilPossibleMitigationsLoaded(); + + // Possible Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' + ); + + // It's a technical preview + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTechPreviewBadge' + ); + + // Should display Edit/Create Component Template Link option + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + + // Should display Edit/Create Ingest Pipeline Link option + await testSubjects.existOrFail('datasetQualityManualMitigationsPipelineAccordion'); + + // Check Component Template URl + const button = await testSubjects.find( + 'datasetQualityManualMitigationsCustomComponentTemplateLink' + ); + const componentTemplateUrl = await button.getAttribute('data-test-url'); + + const integrationSpecificCustomName = `${type}-${nginxAccessDatasetName}@custom`; + + // Should point to component template with @custom as value + expect(componentTemplateUrl).to.be( + `/app/management/data/index_management/component_templates/${integrationSpecificCustomName}` + ); + + const pipelineInputBox = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineName' + ); + const pipelineValue = await pipelineInputBox.getAttribute('value'); + + // Expect Pipeline Name to be default logs for non integrations + expect(pipelineValue).to.be(integrationSpecificCustomName); + + const pipelineLink = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineLink' + ); + + const pipelineLinkURL = await pipelineLink.getAttribute('data-test-url'); + + // Expect the pipeline link to point to the pipeline page with empty pipeline value + expect(pipelineLinkURL).to.be( + `/app/management/ingest/ingest_pipelines/?pipeline=${integrationSpecificCustomName}` + ); + }); }); - describe('field limit exceeded', () => { + describe('past field limit exceeded', () => { it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -287,7 +637,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.closeFlyout(); }); - it('should display the limit when the cause is "field limit exceeded"', async () => { + it('should display the current field limit when the cause is "field limit exceeded"', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, expandedDegradedField: 'cloud', @@ -317,122 +667,191 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('current quality issues', () => { - it('should display issues only from latest backing index when current issues toggle is on', async () => { + describe('current field limit issues', () => { + it('should display increase field limit as a possible mitigation for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); - - expect(currentIssuesToggleState).to.be(false); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - - expect(rows.length).to.eql(3); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); - const newCurrentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - expect(newCurrentIssuesToggleState).to.be(true); + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); - const newRows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + const linkURL = await linkButton.getAttribute('href'); - expect(newRows.length).to.eql(2); + expect(linkURL).to.be( + 'https://www.elastic.co/guide/en/elasticsearch/reference/master/mapping-settings-limit.html' + ); }); - it('should keep the toggle on when url state says so', async () => { + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + expandedDegradedField: 'cloud.project', }); - const currentIssuesToggleState = - await PageObjects.datasetQuality.getQualityIssueSwitchState(); + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); - expect(currentIssuesToggleState).to.be(true); + // Should not display the panel to increase field limit + await testSubjects.missingOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); }); - it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + it('should display additional input fields and button increasing the limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'test_field', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - // Check value in Table - const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const countColumn = table['Docs count']; - expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + // Should display current field limit + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingCurrentLimitFieldText'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const countValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '5' - ); - expect(countValue).to.be(true); - }); + const currentFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingCurrentLimitFieldText' + ); - // Toggle the switch - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + const currentFieldLimitValue = await currentFieldLimitInput.getAttribute('value'); + const currentFieldLimit = parseInt(currentFieldLimitValue, 10); + const currentFieldLimitDisabledStatus = await currentFieldLimitInput.getAttribute( + 'disabled' ); - // Check value in Table - const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const newCountColumn = newTable['Docs count']; - expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + expect(currentFieldLimit).to.be(44); + expect(currentFieldLimitDisabledStatus).to.be('true'); - // Check value in Flyout - await retry.tryForTime(5000, async () => { - const newCountValue = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', - '15' - ); - expect(newCountValue).to.be(true); - }); + // Should display new field limit + await testSubjects.existOrFail( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' + ); + + const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); + const newFieldLimit = parseInt(newFieldLimitValue, 10); + const newFieldLimitDisabledStatus = await newFieldLimitInput.getAttribute('disabled'); + + // Should be 30% more the current limit + const newLimit = Math.round(currentFieldLimit * 1.3); + expect(newFieldLimit).to.be(newLimit); + + // Should display the apply button + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be(null); }); - it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + it('should validate input for new field limit', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', - showCurrentQualityIssues: true, + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + // Should not allow values less than current limit of 44 + await testSubjects.setValue( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText', + '42', + { + clearWithKeyboard: true, + typeCharByChar: true, + } + ); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); + + // The apply button should be active + expect(applyButtonDisabledStatus).to.be('true'); + + const newFieldLimitInput = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingProposedLimitFieldText' ); + const invalidStatus = await newFieldLimitInput.getAttribute('aria-invalid'); + + expect(invalidStatus).to.be('true'); }); - it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDatasetWithLimitDataStreamName, - expandedDegradedField: 'cloud', + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', }); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' ); - await testSubjects.click( - PageObjects.datasetQuality.testSubjectSelectors - .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the success callout + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetSuccessCallout' + ); + + // Should display link to component template edited + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + + const ctLink = await testSubjects.find( + 'datasetQualityDetailsDegradedFlyoutNewLimitSetCheckComponentTemplate' + ); + const ctLinkURL = await ctLink.getAttribute('href'); + + // Should point to the component template page + expect( + ctLinkURL.endsWith( + `/app/management/data/index_management/component_templates/${type}-${nginxAccessDatasetName}@custom` + ) + ).to.be(true); + }); + + // Refresh the time range to get the latest data + await PageObjects.datasetQuality.refreshDetailsPageData(); + + // The page should now handle this as ignore_malformed issue and show a warning + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist' ); + // Should not display the panel to increase field limit await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' ); }); }); @@ -443,6 +862,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { name: degradedDatasetWithLimitDataStreamName, }); await synthtrace.deleteComponentTemplate(customComponentTemplateName); + await PageObjects.observabilityLogsExplorer.uninstallPackage(nginxPkg); + await synthtrace.deleteComponentTemplate(customComponentTemplateNameNginx); }); }); }); From de5db889de39ff32cece9e86af6ca43c76d3b876 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 22 Oct 2024 16:31:07 +0200 Subject: [PATCH 27/28] Fix Editor role permissions and tests --- .../dataset_quality/common/translations.ts | 16 +++++++++++- .../field_limit/message_callout.tsx | 26 ++++++++++++++++--- .../data_stream_details_client.ts | 2 +- .../state_machine.ts | 5 +++- .../types.ts | 2 +- .../data_streams/update_field_limit/index.ts | 17 +++--------- .../dataset_quality/data_stream_settings.ts | 2 +- .../dataset_quality/update_field_limit.ts | 8 +++--- .../dataset_quality/degraded_field_flyout.ts | 23 +++++++++++++++- 9 files changed, 75 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 70c249e4dc9311..1026dd8ea58d37 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -630,15 +630,29 @@ export const fieldLimitMitigationSuccessComponentTemplateLinkText = i18n.transla } ); +export const fieldLimitMitigationPartiallyFailedMessage = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessage', + { + defaultMessage: 'Changes not applied to new data', + } +); + export const fieldLimitMitigationFailedMessage = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessage', { - defaultMessage: 'Changes not applied to new data', + defaultMessage: 'Changes not applied', } ); export const fieldLimitMitigationFailedMessageDescription = i18n.translate( 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationFailedMessageDescription', + { + defaultMessage: 'Failed to set new limit', + } +); + +export const fieldLimitMitigationPartiallyFailedMessageDescription = i18n.translate( + 'xpack.datasetQuality.details.degradedField.possibleMitigation.fieldLimitMitigationPartiallyFailedMessageDescription', { defaultMessage: 'The component template was successfully updated with the new field limit, but the changes were not applied to the most recent backing index. Perform a rollover to apply your changes to new data.', diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx index 8a1e85941ebd18..dded9e4a62eeba 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -10,6 +10,8 @@ import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui'; import { fieldLimitMitigationFailedMessage, fieldLimitMitigationFailedMessageDescription, + fieldLimitMitigationPartiallyFailedMessage, + fieldLimitMitigationPartiallyFailedMessageDescription, fieldLimitMitigationRolloverButton, fieldLimitMitigationSuccessComponentTemplateLinkText, fieldLimitMitigationSuccessMessage, @@ -20,11 +22,16 @@ import { useKibanaContextForPlugin } from '../../../../../utils'; export function MessageCallout() { const { isSavingNewFieldLimitInProgress, newFieldLimitResult } = useDegradedFields(); - const { isComponentTemplateUpdated, isLatestBackingIndexUpdated } = newFieldLimitResult ?? {}; + const { isComponentTemplateUpdated, isLatestBackingIndexUpdated, error } = + newFieldLimitResult ?? {}; const isSuccess = Boolean(isComponentTemplateUpdated) && Boolean(isLatestBackingIndexUpdated); const isPartialSuccess = Boolean(isComponentTemplateUpdated) && !Boolean(isLatestBackingIndexUpdated); + if (error) { + return ; + } + if (!isSavingNewFieldLimitInProgress && isSuccess) { return ; } @@ -76,11 +83,11 @@ export function ManualRolloverCallout() { const { triggerRollover } = useDegradedFields(); return ( -

{fieldLimitMitigationFailedMessageDescription}

+

{fieldLimitMitigationPartiallyFailedMessageDescription}

); } + +export function ErrorCallout() { + return ( + +

{fieldLimitMitigationFailedMessageDescription}

+ + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 40e146794c0c26..827cd4b0a1e493 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -212,7 +212,7 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { { body: JSON.stringify({ newFieldLimit }) } ) .catch((error) => { - throw new DatasetQualityError(`Failed to set new Limit": ${error}`, error); + throw new DatasetQualityError(`Failed to set new Limit: ${error.message}`, error); }); const decodedResponse = decodeOrThrow( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index e71b3467f58136..d762a2f68c0dd4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -389,7 +389,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, onError: { target: 'done', - actions: 'notifySaveNewFieldLimitError', + actions: ['storeErrorResponse', 'notifySaveNewFieldLimitError'], }, }, }, @@ -538,6 +538,9 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( return 'data' in event ? { fieldLimitResponse: event.data } : {}; } ), + storeErrorResponse: assign(() => { + return { fieldLimitResponse: { error: true } }; + }), resetFieldLimitServerResponse: assign(() => ({ newFieldLimit: undefined, fieldLimitResponse: undefined, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index 9f58c14cbcd94a..6906b65240ef94 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -96,7 +96,7 @@ export interface WithNewFieldLimit { } export interface WithNewFieldLimitResponse { - fieldLimitResponse: UpdateFieldLimitResponse; + fieldLimitResponse: UpdateFieldLimitResponse | { error: boolean }; } export type DefaultDatasetQualityDetailsContext = Pick< diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts index 4e6a42248c17b1..f377ea1e2642c8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -6,6 +6,7 @@ */ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { badRequest } from '@hapi/boom'; import { createDatasetQualityESClient } from '../../../utils'; import { updateComponentTemplate } from './update_component_template'; import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; @@ -29,12 +30,7 @@ export async function updateFieldLimit({ }); if (!lastBackingIndexName || !indexTemplate) { - return { - isComponentTemplateUpdated: false, - isLatestBackingIndexUpdated: false, - customComponentTemplateName: '', - error: 'Data stream does not exists', - }; + throw badRequest(`Data stream does not exists. Received value "${dataStream}"`); } const { @@ -43,13 +39,8 @@ export async function updateFieldLimit({ error: errorUpdatingComponentTemplate, } = await updateComponentTemplate({ datasetQualityESClient, indexTemplate, newFieldLimit }); - if (!isComponentTemplateUpdated) { - return { - isComponentTemplateUpdated, - isLatestBackingIndexUpdated: false, - customComponentTemplateName: componentTemplateName, - error: errorUpdatingComponentTemplate, - }; + if (errorUpdatingComponentTemplate) { + throw badRequest(errorUpdatingComponentTemplate); } const { acknowledged: isLatestBackingIndexUpdated, error: errorUpdatingBackingIndex } = diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts index d75b8940987cdf..8b9f60ff1b6676 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -186,7 +186,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); }); - it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly when available for non integration', async () => { + it('returns "createdOn", "integration", "indexTemplate" and "lastBackingIndexName" correctly when available for integration', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, syntheticsDataStreamName diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts index 8f4b58cc758e68..977f9d6f908b1e 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/update_field_limit.ts @@ -98,10 +98,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }, }); - expect(resp.body.isComponentTemplateUpdated).to.be(false); - expect(resp.body.isLatestBackingIndexUpdated).to.be(false); - expect(resp.body.customComponentTemplateName).to.be(''); - expect(resp.body.error).to.be('Data stream does not exists'); + expect(resp.body.statusCode).to.be(400); + expect(resp.body.message).to.be( + `Data stream does not exists. Received value "${invalidDataStreamName}"` + ); }); it('should update last backing index and custom component template', async () => { diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 1bf2d37433855c..8bbd88b7780887 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -755,7 +755,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const newFieldLimitValue = await newFieldLimitInput.getAttribute('value'); const newFieldLimit = parseInt(newFieldLimitValue, 10); - const newFieldLimitDisabledStatus = await newFieldLimitInput.getAttribute('disabled'); // Should be 30% more the current limit const newLimit = Math.round(currentFieldLimit * 1.3); @@ -805,6 +804,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(invalidStatus).to.be('true'); }); + it('should validate and show error callout when API call fails', async () => { + await PageObjects.svlCommonPage.loginWithPrivilegedRole(); + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nginxAccessDataStreamName, + expandedDegradedField: 'cloud.project.id', + }); + + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + + await applyButton.click(); + + await retry.tryForTime(5000, async () => { + // Should display the error callout + await testSubjects.existOrFail('datasetQualityDetailsNewFieldLimitErrorCallout'); + }); + + await PageObjects.svlCommonPage.loginAsAdmin(); + }); + it('should let user increase the field limit for integrations', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: nginxAccessDataStreamName, From 5f3c3710ca7f86d875889bc4ab4ee0474edb0a7c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 22 Oct 2024 16:43:12 +0200 Subject: [PATCH 28/28] Fix Dima's review comments --- .../dataset_quality/data_stream_settings.ts | 11 ++++- .../dataset_quality/integrations.ts | 21 +++++---- .../services/package_api.ts | 47 +++++++++---------- 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts index 8b9f60ff1b6676..8eb8c843fbf21a 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_settings.ts @@ -17,6 +17,7 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont import { SupertestWithRoleScopeType } from '../../../services'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); const roleScopedSupertest = getService('roleScopedSupertest'); const synthtrace = getService('logsSynthtraceEsClient'); const esClient = getService('es'); @@ -53,9 +54,11 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { } describe('Dataset quality settings', function () { + let adminRoleAuthc: RoleCredentials; let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( 'admin', { @@ -65,6 +68,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ); }); + after(async () => { + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + it('returns only privileges if matching data stream is not available', async () => { const nonExistentDataSet = 'Non-existent'; const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; @@ -156,7 +163,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { describe('gets the data stream settings for integrations', () => { before(async () => { await packageApi.installPackage({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + roleAuthc: adminRoleAuthc, pkg: syntheticsDataset, }); await synthtrace.index([ @@ -181,7 +188,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { after(async () => { await synthtrace.clean(); await packageApi.uninstallPackage({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + roleAuthc: adminRoleAuthc, pkg: syntheticsDataset, }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts index d832a723959809..8290f52729ba81 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts @@ -12,6 +12,7 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont import { SupertestWithRoleScopeType } from '../../../services'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); const roleScopedSupertest = getService('roleScopedSupertest'); const packageApi = getService('packageApi'); @@ -44,9 +45,11 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { } describe('Integrations', () => { + let adminRoleAuthc: RoleCredentials; let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( 'admin', { @@ -56,12 +59,16 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ); }); + after(async () => { + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + describe('gets the installed integrations', () => { before(async () => { await Promise.all( integrationPackages.map((pkg) => packageApi.installPackage({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + roleAuthc: adminRoleAuthc, pkg, }) ) @@ -86,10 +93,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { async () => await Promise.all( integrationPackages.map((pkg) => - packageApi.uninstallPackage({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, - pkg, - }) + packageApi.uninstallPackage({ roleAuthc: adminRoleAuthc, pkg }) ) ) ); @@ -99,10 +103,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { before(async () => { await Promise.all( customIntegrations.map((customIntegration: CustomIntegration) => - packageApi.installCustomIntegration({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, - customIntegration, - }) + packageApi.installCustomIntegration({ roleAuthc: adminRoleAuthc, customIntegration }) ) ); }); @@ -126,7 +127,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await Promise.all( customIntegrations.map((customIntegration: CustomIntegration) => packageApi.uninstallPackage({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + roleAuthc: adminRoleAuthc, pkg: customIntegration.integrationName, }) ) diff --git a/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts b/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts index c97b674e7b3a77..4406147f6b7564 100644 --- a/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts +++ b/x-pack/test/api_integration/deployment_agnostic/services/package_api.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { SupertestWithRoleScopeType } from '.'; +import { RoleCredentials } from '@kbn/ftr-common-functional-services'; +import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context'; export interface CustomIntegration { integrationName: string; @@ -17,52 +18,50 @@ export interface IntegrationDataset { type: 'logs' | 'metrics' | 'synthetics' | 'traces'; } -export function PackageApiProvider() { +export function PackageApiProvider({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + return { async installCustomIntegration({ - roleScopedSupertestWithCookieCredentials, + roleAuthc, customIntegration, }: { - roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + roleAuthc: RoleCredentials; customIntegration: CustomIntegration; }) { const { integrationName, datasets } = customIntegration; - const { body } = await roleScopedSupertestWithCookieCredentials + const { body } = await supertestWithoutAuth .post(`/api/fleet/epm/custom_integrations`) + .set(roleAuthc.apiKeyHeader) + .set(samlAuth.getInternalRequestHeader()) .send({ integrationName, datasets }); return body; }, - async installPackage({ - roleScopedSupertestWithCookieCredentials, - pkg, - }: { - roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; - pkg: string; - }) { + async installPackage({ roleAuthc, pkg }: { roleAuthc: RoleCredentials; pkg: string }) { const { body: { item: { latestVersion: version }, }, - } = await roleScopedSupertestWithCookieCredentials + } = await supertestWithoutAuth .get(`/api/fleet/epm/packages/${pkg}`) + .set(roleAuthc.apiKeyHeader) + .set(samlAuth.getInternalRequestHeader()) .send({ force: true }); - const { body } = await roleScopedSupertestWithCookieCredentials + const { body } = await supertestWithoutAuth .post(`/api/fleet/epm/packages/${pkg}/${version}`) + .set(roleAuthc.apiKeyHeader) + .set(samlAuth.getInternalRequestHeader()) .send({ force: true }); return body; }, - async uninstallPackage({ - roleScopedSupertestWithCookieCredentials, - pkg, - }: { - roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; - pkg: string; - }) { - const { body } = await roleScopedSupertestWithCookieCredentials.delete( - `/api/fleet/epm/packages/${pkg}` - ); + async uninstallPackage({ roleAuthc, pkg }: { roleAuthc: RoleCredentials; pkg: string }) { + const { body } = await supertestWithoutAuth + .delete(`/api/fleet/epm/packages/${pkg}`) + .set(roleAuthc.apiKeyHeader) + .set(samlAuth.getInternalRequestHeader()); return body; }, };