[8.0] [UA] Remove deprecation logs created by Elastic products (#121174…

…) (#121866)

src/core/public/doc_links/doc_links_service.ts

@@ -282,6 +282,7 @@ export class DocLinksService {
  migrationApiDeprecation: `${ELASTICSEARCH_DOCS}migration-api-deprecation.html`,
  nodeRoles: `${ELASTICSEARCH_DOCS}modules-node.html#node-roles`,
  releaseHighlights: `${ELASTICSEARCH_DOCS}release-highlights.html`,
+ version8ReleaseHighlights: `${ELASTIC_WEBSITE_URL}guide/en/elastic-stack/8.0/elastic-stack-highlights.html`,
  remoteClusters: `${ELASTICSEARCH_DOCS}remote-clusters.html`,
  remoteClustersProxy: `${ELASTICSEARCH_DOCS}remote-clusters.html#proxy-mode`,
  remoteClusersProxySettings: `${ELASTICSEARCH_DOCS}remote-clusters-settings.html#remote-cluster-proxy-settings`,

x-pack/plugins/upgrade_assistant/__jest__/client_integration/es_deprecation_logs/es_deprecation_logs.test.tsx

@@ -16,6 +16,8 @@ import {
  DEPRECATION_LOGS_INDEX,
  DEPRECATION_LOGS_SOURCE_ID,
  DEPRECATION_LOGS_COUNT_POLL_INTERVAL_MS,
+ APPS_WITH_DEPRECATION_LOGS,
+ DEPRECATION_LOGS_ORIGIN_FIELD,
 } from '../../../common/constants';
 
 // Once the logs team register the kibana locators in their app, we should be able
@@ -171,9 +173,15 @@ describe('ES deprecation logs', () => {
  component.update();
 
  expect(exists('viewObserveLogs')).toBe(true);
- expect(find('viewObserveLogs').props().href).toBe(
- `/app/logs/stream?sourceId=${DEPRECATION_LOGS_SOURCE_ID}&logPosition=(end:now,start:'${MOCKED_TIME}')`
+ const sourceId = DEPRECATION_LOGS_SOURCE_ID;
+ const logPosition = `(end:now,start:'${MOCKED_TIME}')`;
+ const logFilter = encodeURI(
+ `(language:kuery,query:'not ${DEPRECATION_LOGS_ORIGIN_FIELD} : (${APPS_WITH_DEPRECATION_LOGS.join(
+ ' or '
+ )})')`
  );
+ const queryParams = `sourceId=${sourceId}&logPosition=${logPosition}&logFilter=${logFilter}`;
+ expect(find('viewObserveLogs').props().href).toBe(`/app/logs/stream?${queryParams}`);
  });
 
  test(`Doesn't show observability app link if infra app is not available`, async () => {
@@ -197,8 +205,18 @@ describe('ES deprecation logs', () => {
 
  const decodedUrl = decodeURIComponent(find('viewDiscoverLogs').props().href);
  expect(decodedUrl).toContain('discoverUrl');
- ['"language":"kuery"', '"query":"@timestamp+>'].forEach((param) => {
- expect(decodedUrl).toContain(param);
+ [
+ '"language":"kuery"',
+ '"query":"@timestamp+>',
+ 'filters=',
+ DEPRECATION_LOGS_ORIGIN_FIELD,
+ ...APPS_WITH_DEPRECATION_LOGS,
+ ].forEach((param) => {
+ try {
+ expect(decodedUrl).toContain(param);
+ } catch (e) {
+ throw new Error(`Expected [${param}] not found in ${decodedUrl}`);
+ }
  });
  });
  });

x-pack/plugins/upgrade_assistant/__jest__/client_integration/helpers/app_context.mock.ts

@@ -22,10 +22,29 @@ import { breadcrumbService } from '../../../public/application/lib/breadcrumbs';
 import { dataPluginMock } from '../../../../../../src/plugins/data/public/mocks';
 import { cloudMock } from '../../../../../../x-pack/plugins/cloud/public/mocks';
 
+const data = dataPluginMock.createStartContract();
+const dataViews = { ...data.dataViews };
+const findDataView = (id: string) =>
+ Promise.resolve([
+ {
+ id,
+ title: id,
+ getFieldByName: jest.fn((name: string) => ({
+ name,
+ })),
+ },
+ ]);
+
 const servicesMock = {
  api: apiService,
  breadcrumbs: breadcrumbService,
- data: dataPluginMock.createStartContract(),
+ data: {
+ ...data,
+ dataViews: {
+ ...dataViews,
+ find: findDataView,
+ },
+ },
 };
 
 // We'll mock these values to avoid testing the locators themselves.

x-pack/plugins/upgrade_assistant/common/constants.ts

@@ -41,3 +41,23 @@ export const CLUSTER_UPGRADE_STATUS_POLL_INTERVAL_MS = 45000;
 export const CLOUD_BACKUP_STATUS_POLL_INTERVAL_MS = 60000;
 export const DEPRECATION_LOGS_COUNT_POLL_INTERVAL_MS = 15000;
 export const SYSTEM_INDICES_MIGRATION_POLL_INTERVAL_MS = 15000;
+
+/**
+ * List of Elastic apps that potentially can generate deprecation logs.
+ * We want to filter those out for our users so they only see deprecation logs
+ * that _they_ are generating.
+ */
+export const APPS_WITH_DEPRECATION_LOGS = [
+ 'kibana',
+ 'cloud',
+ 'logstash',
+ 'beats',
+ 'fleet',
+ 'ml',
+ 'security',
+ 'observability',
+ 'enterprise-search',
+];
+
+// The field that will indicate which elastic product generated the deprecation log
+export const DEPRECATION_LOGS_ORIGIN_FIELD = 'elasticsearch.elastic_product_origin';

x-pack/plugins/upgrade_assistant/public/application/components/es_deprecation_logs/fix_deprecation_logs/external_links.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { getDeprecationIndexPatternId } from './external_links';
+import { getDeprecationDataView } from './external_links';
 
 import { DEPRECATION_LOGS_INDEX_PATTERN } from '../../../../../common/constants';
 import { dataPluginMock, Start } from '../../../../../../../../src/plugins/data/public/mocks';
@@ -17,32 +17,32 @@ describe('External Links', () => {
  dataService = dataPluginMock.createStartContract();
  });
 
- describe('getDeprecationIndexPatternId', () => {
- it('creates new index pattern if doesnt exist', async () => {
+ describe('getDeprecationDataView', () => {
+ it('creates new data view if doesnt exist', async () => {
  dataService.dataViews.find = jest.fn().mockResolvedValue([]);
  dataService.dataViews.createAndSave = jest.fn().mockResolvedValue({ id: '123-456' });
 
- const indexPatternId = await getDeprecationIndexPatternId(dataService);
+ const dataViewId = (await getDeprecationDataView(dataService)).id;
 
- expect(indexPatternId).toBe('123-456');
+ expect(dataViewId).toBe('123-456');
  // prettier-ignore
  expect(dataService.dataViews.createAndSave).toHaveBeenCalledWith({
  title: DEPRECATION_LOGS_INDEX_PATTERN,
  allowNoIndex: true,
  }, false, true);
  });
 
- it('uses existing index pattern if it already exists', async () => {
+ it('uses existing data view if it already exists', async () => {
  dataService.dataViews.find = jest.fn().mockResolvedValue([
  {
  id: '123-456',
  title: DEPRECATION_LOGS_INDEX_PATTERN,
  },
  ]);
 
- const indexPatternId = await getDeprecationIndexPatternId(dataService);
+ const dataViewId = await (await getDeprecationDataView(dataService)).id;
 
- expect(indexPatternId).toBe('123-456');
+ expect(dataViewId).toBe('123-456');
  expect(dataService.dataViews.find).toHaveBeenCalledWith(DEPRECATION_LOGS_INDEX_PATTERN);
  });
  });

x-pack/plugins/upgrade_assistant/public/application/components/es_deprecation_logs/fix_deprecation_logs/external_links.tsx

@@ -7,11 +7,16 @@
 
 import { encode } from 'rison-node';
 import React, { FunctionComponent, useState, useEffect } from 'react';
+import { buildPhrasesFilter, PhraseFilter } from '@kbn/es-query';
 
 import { FormattedMessage } from '@kbn/i18n-react';
 import { METRIC_TYPE } from '@kbn/analytics';
 import { EuiLink, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiPanel, EuiText } from '@elastic/eui';
 
+import {
+ APPS_WITH_DEPRECATION_LOGS,
+ DEPRECATION_LOGS_ORIGIN_FIELD,
+} from '../../../../../common/constants';
 import { DataPublicPluginStart } from '../../../../shared_imports';
 import { useAppContext } from '../../../app_context';
 import {
@@ -29,32 +34,32 @@ interface Props {
  checkpoint: string;
 }
 
-export const getDeprecationIndexPatternId = async (dataService: DataPublicPluginStart) => {
+export const getDeprecationDataView = async (dataService: DataPublicPluginStart) => {
  const results = await dataService.dataViews.find(DEPRECATION_LOGS_INDEX_PATTERN);
  // Since the find might return also results with wildcard matchers we need to find the
  // index pattern that has an exact match with our title.
- const deprecationIndexPattern = results.find(
+ const deprecationDataView = results.find(
  (result) => result.title === DEPRECATION_LOGS_INDEX_PATTERN
  );
 
- if (deprecationIndexPattern) {
- return deprecationIndexPattern.id;
+ if (deprecationDataView) {
+ return deprecationDataView;
  } else {
- // When creating the index pattern, we need to be careful when creating an indexPattern
+ // When creating the data view, we need to be careful when creating a data view
  // for an index that doesnt exist. Since the deprecation logs data stream is only created
  // when a deprecation log is indexed it could be possible that it might not exist at the
  // time we need to render the DiscoveryAppLink.
- // So in order to avoid those errors we need to make sure that the indexPattern is created
+ // So in order to avoid those errors we need to make sure that the data view is created
  // with allowNoIndex and that we skip fetching fields to from the source index.
  const override = false;
  const skipFetchFields = true;
  // prettier-ignore
- const newIndexPattern = await dataService.dataViews.createAndSave({
+ const newDataView = await dataService.dataViews.createAndSave({
  title: DEPRECATION_LOGS_INDEX_PATTERN,
  allowNoIndex: true,
  }, override, skipFetchFields);
 
- return newIndexPattern.id;
+ return newDataView;
  }
 };
 
@@ -68,19 +73,29 @@ const DiscoverAppLink: FunctionComponent<Props> = ({ checkpoint }) => {
 
  useEffect(() => {
  const getDiscoveryUrl = async () => {
- const indexPatternId = await getDeprecationIndexPatternId(dataService);
  const locator = share.url.locators.get('DISCOVER_APP_LOCATOR');
-
  if (!locator) {
  return;
  }
 
- const url = await locator.getUrl({
- indexPatternId,
+ const dataView = await getDeprecationDataView(dataService);
+ const field = dataView.getFieldByName(DEPRECATION_LOGS_ORIGIN_FIELD);
+
+ let filters: PhraseFilter[] = [];
+
+ if (field !== undefined) {
+ const filter = buildPhrasesFilter(field!, [...APPS_WITH_DEPRECATION_LOGS], dataView);
+ filter.meta.negate = true;
+ filters = [filter];
+ }
+
+ const url = await locator?.getUrl({
+ indexPatternId: dataView.id,
  query: {
  language: 'kuery',
  query: `@timestamp > "${checkpoint}"`,
  },
+ filters,
  });
 
  setDiscoveryUrl(url);
@@ -89,6 +104,10 @@ const DiscoverAppLink: FunctionComponent<Props> = ({ checkpoint }) => {
  getDiscoveryUrl();
  }, [dataService, checkpoint, share.url.locators]);
 
+ if (discoveryUrl === undefined) {
+ return null;
+ }
+
  return (
  // eslint-disable-next-line @elastic/eui/href-or-on-click
  <EuiLink
@@ -112,11 +131,21 @@ const ObservabilityAppLink: FunctionComponent<Props> = ({ checkpoint }) => {
  core: { http },
  },
  } = useAppContext();
- const logStreamUrl = http?.basePath?.prepend(
- `/app/logs/stream?sourceId=${DEPRECATION_LOGS_SOURCE_ID}&logPosition=(end:now,start:${encode(
- checkpoint
- )})`
+
+ // Ideally we don't want to hardcode the path to the Log Stream app and use the UrlService.locator instead.
+ // Issue opened: https:/elastic/kibana/issues/104855
+ const streamAppPath = '/app/logs/stream';
+
+ const sourceId = DEPRECATION_LOGS_SOURCE_ID;
+ const logPosition = `(end:now,start:${encode(checkpoint)})`;
+ const logFilter = encodeURI(
+ `(language:kuery,query:'not ${DEPRECATION_LOGS_ORIGIN_FIELD} : (${APPS_WITH_DEPRECATION_LOGS.join(
+ ' or '
+ )})')`
  );
+ const queryParams = `sourceId=${sourceId}&logPosition=${logPosition}&logFilter=${logFilter}`;
+
+ const logStreamUrl = http?.basePath?.prepend(`${streamAppPath}?${queryParams}`);
 
  return (
  // eslint-disable-next-line @elastic/eui/href-or-on-click

x-pack/plugins/upgrade_assistant/public/application/components/overview/overview.tsx

@@ -88,7 +88,7 @@ export const Overview = withRouter(({ history }: RouteComponentProps) => {
  ]}
  >
  <EuiText data-test-subj="whatsNewLink">
- <EuiLink href={docLinks.links.elasticsearch.releaseHighlights} target="_blank">
+ <EuiLink href={docLinks.links.elasticsearch.version8ReleaseHighlights} target="_blank">
  <FormattedMessage
  id="xpack.upgradeAssistant.overview.whatsNewLink"
  defaultMessage="What's new in 8.x?"

x-pack/plugins/upgrade_assistant/server/routes/deprecation_logging.ts

@@ -5,8 +5,13 @@
  * 2.0.
  */
 
+import moment from 'moment-timezone';
 import { schema } from '@kbn/config-schema';
-import { API_BASE_PATH } from '../../common/constants';
+import {
+ API_BASE_PATH,
+ APPS_WITH_DEPRECATION_LOGS,
+ DEPRECATION_LOGS_ORIGIN_FIELD,
+} from '../../common/constants';
 
 import {
  getDeprecationLoggingStatus,
@@ -104,13 +109,25 @@ export function registerDeprecationLoggingRoutes({
  return response.ok({ body: { count: 0 } });
  }
 
+ const now = moment().toISOString();
+
  const { body } = await client.asCurrentUser.count({
  index: DEPRECATION_LOGS_INDEX,
  body: {
  query: {
- range: {
- '@timestamp': {
- gte: request.query.from,
+ bool: {
+ must: {
+ range: {
+ '@timestamp': {
+ gte: request.query.from,
+ lte: now,
+ },
+ },
+ },
+ must_not: {
+ terms: {
+ [DEPRECATION_LOGS_ORIGIN_FIELD]: [...APPS_WITH_DEPRECATION_LOGS],
+ },
  },
  },
  },