[EPM] Conditionally generate ES index pattern name based on dataset_i…

…s_prefix (elastic#89870) * Explicitly generate ES index pattern name. * Adjust tests. * Adjust and reenable tests. * Set template priority based on dataset_is_prefix * Refactor indexPatternName -> templateIndexPattern * Add unit tests. * Use more realistic index pattern in test. * Fix unit test. * Add unit test for installTemplate(). Co-authored-by: Kibana Machine <[email protected]>
skh · Feb 8, 2021 · a1ff935 · a1ff935
1 parent 7d220d5
commit a1ff935
Show file tree

Hide file tree

Showing 7 changed files with 268 additions and 34 deletions.
diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -221,6 +221,7 @@ export interface RegistryDataStream {
  path: string;
  ingest_pipeline: string;
  elasticsearch?: RegistryElasticsearch;
+ dataset_is_prefix?: boolean;
 }
 
 export interface RegistryElasticsearch {

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RegistryDataStream } from '../../../../types';
+import { Field } from '../../fields/field';
+
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
+import { installTemplate } from './install';
+
+test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
+ const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
+ const fields: Field[] = [];
+ const dataStreamDatasetIsPrefixUnset = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ } as RegistryDataStream;
+ const pkg = {
+ name: 'package',
+ version: '0.0.1',
+ };
+ const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
+ const templatePriorityDatasetIsPrefixUnset = 200;
+ await installTemplate({
+ callCluster,
+ fields,
+ dataStream: dataStreamDatasetIsPrefixUnset,
+ packageVersion: pkg.version,
+ packageName: pkg.name,
+ });
+ // @ts-ignore
+ const sentTemplate = callCluster.mock.calls[0][1].body;
+ expect(sentTemplate).toBeDefined();
+ expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset);
+ expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
+});
+
+test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
+ const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
+ const fields: Field[] = [];
+ const dataStreamDatasetIsPrefixFalse = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ dataset_is_prefix: false,
+ } as RegistryDataStream;
+ const pkg = {
+ name: 'package',
+ version: '0.0.1',
+ };
+ const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
+ const templatePriorityDatasetIsPrefixFalse = 200;
+ await installTemplate({
+ callCluster,
+ fields,
+ dataStream: dataStreamDatasetIsPrefixFalse,
+ packageVersion: pkg.version,
+ packageName: pkg.name,
+ });
+ // @ts-ignore
+ const sentTemplate = callCluster.mock.calls[0][1].body;
+ expect(sentTemplate).toBeDefined();
+ expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse);
+ expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
+});
+
+test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
+ const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
+ const fields: Field[] = [];
+ const dataStreamDatasetIsPrefixTrue = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ dataset_is_prefix: true,
+ } as RegistryDataStream;
+ const pkg = {
+ name: 'package',
+ version: '0.0.1',
+ };
+ const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
+ const templatePriorityDatasetIsPrefixTrue = 150;
+ await installTemplate({
+ callCluster,
+ fields,
+ dataStream: dataStreamDatasetIsPrefixTrue,
+ packageVersion: pkg.version,
+ packageName: pkg.name,
+ });
+ // @ts-ignore
+ const sentTemplate = callCluster.mock.calls[0][1].body;
+ expect(sentTemplate).toBeDefined();
+ expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue);
+ expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
+});
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
@@ -17,7 +17,13 @@ import {
 import { CallESAsCurrentUser } from '../../../../types';
 import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
-import { generateMappings, generateTemplateName, getTemplate } from './template';
+import {
+ generateMappings,
+ generateTemplateName,
+ generateTemplateIndexPattern,
+ getTemplate,
+ getTemplatePriority,
+} from './template';
 import { getAsset, getPathParts } from '../../archive';
 import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install';
 
@@ -293,6 +299,9 @@ export async function installTemplate({
 }): Promise<TemplateRef> {
  const mappings = generateMappings(processFields(fields));
  const templateName = generateTemplateName(dataStream);
+ const templateIndexPattern = generateTemplateIndexPattern(dataStream);
+ const templatePriority = getTemplatePriority(dataStream);
+
  let pipelineName;
  if (dataStream.ingest_pipeline) {
  pipelineName = getPipelineNameForInstallation({
@@ -310,11 +319,12 @@ export async function installTemplate({
 
  const template = getTemplate({
  type: dataStream.type,
- templateName,
+ templateIndexPattern,
  mappings,
  pipelineName,
  packageName,
  composedOfTemplates,
+ templatePriority,
  ilmPolicy: dataStream.ilm_policy,
  hidden: dataStream.hidden,
  });

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -8,8 +8,14 @@
 import { readFileSync } from 'fs';
 import { safeLoad } from 'js-yaml';
 import path from 'path';
+import { RegistryDataStream } from '../../../../types';
 import { Field, processFields } from '../../fields/field';
-import { generateMappings, getTemplate } from './template';
+import {
+ generateMappings,
+ getTemplate,
+ getTemplatePriority,
+ generateTemplateIndexPattern,
+} from './template';
 
 // Add our own serialiser to just do JSON.stringify
 expect.addSnapshotSerializer({
@@ -23,27 +29,29 @@ expect.addSnapshotSerializer({
 });
 
 test('get template', () => {
- const templateName = 'logs-nginx-access-abcd';
+ const templateIndexPattern = 'logs-nginx.access-abcd-*';
 
  const template = getTemplate({
  type: 'logs',
- templateName,
+ templateIndexPattern,
  packageName: 'nginx',
  mappings: { properties: {} },
  composedOfTemplates: [],
+ templatePriority: 200,
  });
- expect(template.index_patterns).toStrictEqual([`${templateName}-*`]);
+ expect(template.index_patterns).toStrictEqual([templateIndexPattern]);
 });
 
 test('adds composed_of correctly', () => {
  const composedOfTemplates = ['component1', 'component2'];
 
  const template = getTemplate({
  type: 'logs',
- templateName: 'name',
+ templateIndexPattern: 'name-*',
  packageName: 'nginx',
  mappings: { properties: {} },
  composedOfTemplates,
+ templatePriority: 200,
  });
  expect(template.composed_of).toStrictEqual(composedOfTemplates);
 });
@@ -53,35 +61,36 @@ test('adds empty composed_of correctly', () => {
 
  const template = getTemplate({
  type: 'logs',
- templateName: 'name',
+ templateIndexPattern: 'name-*',
  packageName: 'nginx',
  mappings: { properties: {} },
  composedOfTemplates,
+ templatePriority: 200,
  });
  expect(template.composed_of).toStrictEqual(composedOfTemplates);
 });
 
 test('adds hidden field correctly', () => {
- const templateWithHiddenName = 'logs-nginx-access-abcd';
+ const templateIndexPattern = 'logs-nginx.access-abcd-*';
 
  const templateWithHidden = getTemplate({
  type: 'logs',
- templateName: templateWithHiddenName,
+ templateIndexPattern,
  packageName: 'nginx',
  mappings: { properties: {} },
  composedOfTemplates: [],
+ templatePriority: 200,
  hidden: true,
  });
  expect(templateWithHidden.data_stream.hidden).toEqual(true);
 
- const templateWithoutHiddenName = 'logs-nginx-access-efgh';
-
  const templateWithoutHidden = getTemplate({
  type: 'logs',
- templateName: templateWithoutHiddenName,
+ templateIndexPattern,
  packageName: 'nginx',
  mappings: { properties: {} },
  composedOfTemplates: [],
+ templatePriority: 200,
  });
  expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
 });
@@ -95,10 +104,11 @@ test('tests loading base.yml', () => {
  const mappings = generateMappings(processedFields);
  const template = getTemplate({
  type: 'logs',
- templateName: 'foo',
+ templateIndexPattern: 'foo-*',
  packageName: 'nginx',
  mappings,
  composedOfTemplates: [],
+ templatePriority: 200,
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -113,10 +123,11 @@ test('tests loading coredns.logs.yml', () => {
  const mappings = generateMappings(processedFields);
  const template = getTemplate({
  type: 'logs',
- templateName: 'foo',
+ templateIndexPattern: 'foo-*',
  packageName: 'coredns',
  mappings,
  composedOfTemplates: [],
+ templatePriority: 200,
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -131,10 +142,11 @@ test('tests loading system.yml', () => {
  const mappings = generateMappings(processedFields);
  const template = getTemplate({
  type: 'metrics',
- templateName: 'whatsthis',
+ templateIndexPattern: 'whatsthis-*',
  packageName: 'system',
  mappings,
  composedOfTemplates: [],
+ templatePriority: 200,
  });
 
  expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -520,3 +532,62 @@ test('tests constant_keyword field type handling', () => {
  const mappings = generateMappings(processedFields);
  expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping));
 });
+
+test('tests priority and index pattern for data stream without dataset_is_prefix', () => {
+ const dataStreamDatasetIsPrefixUnset = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ } as RegistryDataStream;
+ const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
+ const templatePriorityDatasetIsPrefixUnset = 200;
+ const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset);
+ const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset);
+
+ expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset);
+ expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset);
+});
+
+test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => {
+ const dataStreamDatasetIsPrefixFalse = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ dataset_is_prefix: false,
+ } as RegistryDataStream;
+ const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
+ const templatePriorityDatasetIsPrefixFalse = 200;
+ const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse);
+ const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse);
+
+ expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse);
+ expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse);
+});
+
+test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => {
+ const dataStreamDatasetIsPrefixTrue = {
+ type: 'metrics',
+ dataset: 'package.dataset',
+ title: 'test data stream',
+ release: 'experimental',
+ package: 'package',
+ path: 'path',
+ ingest_pipeline: 'default',
+ dataset_is_prefix: true,
+ } as RegistryDataStream;
+ const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
+ const templatePriorityDatasetIsPrefixTrue = 150;
+ const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue);
+ const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue);
+
+ expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue);
+ expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue);
+});