[Ingest Manager] Match package spec dataset->data_stream and `con…

…fig_templates`->`policy_templates` renaming (#78699) * Match elastic/package-spec#24 `datasets`->`data_streams` property renaming * Match elastic/package-spec#24 `datasets.name`->`data_streams.dataset` property renaming * Match elastic/package-spec#24 `/dataset`->`/data_stream` directory renaming * Match elastic/package-spec#50 `config_templates`->`policy_templates` property renaming * Update API integration test fixtures (test packages) * Temporarily skip API integration tests * Temporarily skip more API integration tests * Pin to custom docker image, unskip test suites, clean up broken icon paths in test package manifests * Skip the only (yay!) failing test suite * Revert "Skip the only (yay!) failing test suite" This reverts commit 3db32e2. * Re-skip tests and revert docker image Co-authored-by: Elastic Machine <[email protected]>
elastic · Oct 1, 2020 · 6d8f74a · 6d8f74a
1 parent 085f8a1
commit 6d8f74a
Show file tree

Hide file tree

Showing 84 changed files with 254 additions and 268 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json b/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json
@@ -1543,7 +1543,7 @@
  }
  },
  "format_version": "1.0.0",
- "datasets": [
+ "data_streams": [
  {
  "title": "CoreDNS logs",
  "name": "log",
@@ -1764,7 +1764,7 @@
  ]
  }
  },
- "datasets": [
+ "data_streams": [
  {
  "id": "endpoint",
  "title": "Endpoint Events",
@@ -3961,7 +3961,7 @@
  "format_version": {
  "type": "string"
  },
- "datasets": {
+ "data_streams": {
  "type": "array",
  "items": {
  "type": "object",

diff --git a/x-pack/plugins/ingest_manager/common/services/limited_package.ts b/x-pack/plugins/ingest_manager/common/services/limited_package.ts
@@ -7,7 +7,7 @@ import { PackageInfo, AgentPolicy, PackagePolicy } from '../types';
 
 // Assume packages only ever include 1 config template for now
 export const isPackageLimited = (packageInfo: PackageInfo): boolean => {
- return packageInfo.config_templates?.[0]?.multiple === false;
+ return packageInfo.policy_templates?.[0]?.multiple === false;
 };
 
 export const doesAgentPolicyAlreadyIncludePackage = (

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts
@@ -34,14 +34,14 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  describe('packageToPackagePolicyInputs', () => {
  it('returns empty array for packages with no config templates', () => {
  expect(packageToPackagePolicyInputs(mockPackage)).toEqual([]);
- expect(packageToPackagePolicyInputs({ ...mockPackage, config_templates: [] })).toEqual([]);
+ expect(packageToPackagePolicyInputs({ ...mockPackage, policy_templates: [] })).toEqual([]);
  });
 
  it('returns empty array for packages with a config template but no inputs', () => {
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- config_templates: [{ inputs: [] }],
+ policy_templates: [{ inputs: [] }],
  } as unknown) as PackageInfo)
  ).toEqual([]);
  });
@@ -50,13 +50,13 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- config_templates: [{ inputs: [{ type: 'foo' }] }],
+ policy_templates: [{ inputs: [{ type: 'foo' }] }],
  } as unknown) as PackageInfo)
  ).toEqual([{ type: 'foo', enabled: true, streams: [] }]);
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- config_templates: [{ inputs: [{ type: 'foo' }, { type: 'bar' }] }],
+ policy_templates: [{ inputs: [{ type: 'foo' }, { type: 'bar' }] }],
  } as unknown) as PackageInfo)
  ).toEqual([
  { type: 'foo', enabled: true, streams: [] },
@@ -68,12 +68,12 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- datasets: [
- { type: 'logs', name: 'foo', streams: [{ input: 'foo' }] },
- { type: 'logs', name: 'bar', streams: [{ input: 'bar' }] },
- { type: 'logs', name: 'bar2', streams: [{ input: 'bar' }] },
+ data_streams: [
+ { type: 'logs', dataset: 'foo', streams: [{ input: 'foo' }] },
+ { type: 'logs', dataset: 'bar', streams: [{ input: 'bar' }] },
+ { type: 'logs', dataset: 'bar2', streams: [{ input: 'bar' }] },
  ],
- config_templates: [
+ policy_templates: [
  {
  inputs: [{ type: 'foo' }, { type: 'bar' }],
  },
@@ -102,15 +102,15 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- datasets: [
+ data_streams: [
  {
  type: 'logs',
- name: 'foo',
+ dataset: 'foo',
  streams: [{ input: 'foo', vars: [{ default: 'foo-var-value', name: 'var-name' }] }],
  },
  {
  type: 'logs',
- name: 'bar',
+ dataset: 'bar',
  streams: [
  {
  input: 'bar',
@@ -120,7 +120,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  },
  {
  type: 'logs',
- name: 'bar2',
+ dataset: 'bar2',
  streams: [
  {
  input: 'bar',
@@ -129,7 +129,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  ],
  },
  ],
- config_templates: [
+ policy_templates: [
  {
  inputs: [{ type: 'foo' }, { type: 'bar' }],
  },
@@ -173,15 +173,15 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  expect(
  packageToPackagePolicyInputs(({
  ...mockPackage,
- datasets: [
+ data_streams: [
  {
  type: 'logs',
- name: 'foo',
+ dataset: 'foo',
  streams: [{ input: 'foo', vars: [{ default: 'foo-var-value', name: 'var-name' }] }],
  },
  {
  type: 'logs',
- name: 'bar',
+ dataset: 'bar',
  streams: [
  {
  input: 'bar',
@@ -191,7 +191,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  },
  {
  type: 'logs',
- name: 'bar2',
+ dataset: 'bar2',
  streams: [
  {
  input: 'bar',
@@ -201,7 +201,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  },
  {
  type: 'logs',
- name: 'disabled',
+ dataset: 'disabled',
  streams: [
  {
  input: 'with-disabled-streams',
@@ -212,7 +212,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  },
  {
  type: 'logs',
- name: 'disabled2',
+ dataset: 'disabled2',
  streams: [
  {
  input: 'with-disabled-streams',
@@ -221,7 +221,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  ],
  },
  ],
- config_templates: [
+ policy_templates: [
  {
  inputs: [
  {
@@ -372,13 +372,13 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
  });
  });
  it('returns package policy with inputs', () => {
- const mockPackageWithConfigTemplates = ({
+ const mockPackageWithPolicyTemplates = ({
  ...mockPackage,
- config_templates: [{ inputs: [{ type: 'foo' }] }],
+ policy_templates: [{ inputs: [{ type: 'foo' }] }],
  } as unknown) as PackageInfo;
 
  expect(
- packageToPackagePolicy(mockPackageWithConfigTemplates, '1', '2', 'default', 'pkgPolicy-1')
+ packageToPackagePolicy(mockPackageWithPolicyTemplates, '1', '2', 'default', 'pkgPolicy-1')
  ).toEqual({
  policy_id: '1',
  namespace: 'default',

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts
@@ -5,7 +5,7 @@
  */
 import {
  PackageInfo,
- RegistryConfigTemplate,
+ RegistryPolicyTemplate,
  RegistryVarsEntry,
  RegistryStream,
  PackagePolicy,
@@ -22,14 +22,14 @@ const getStreamsForInputType = (
 ): Array<RegistryStream & { data_stream: { type: string; dataset: string } }> => {
  const streams: Array<RegistryStream & { data_stream: { type: string; dataset: string } }> = [];
 
- (packageInfo.datasets || []).forEach((dataset) => {
- (dataset.streams || []).forEach((stream) => {
+ (packageInfo.data_streams || []).forEach((dataStream) => {
+ (dataStream.streams || []).forEach((stream) => {
  if (stream.input === inputType) {
  streams.push({
  ...stream,
  data_stream: {
- type: dataset.type,
- dataset: dataset.name,
+ type: dataStream.type,
+ dataset: dataStream.dataset,
  },
  });
  }
@@ -46,9 +46,9 @@ export const packageToPackagePolicyInputs = (packageInfo: PackageInfo): PackageP
  const inputs: PackagePolicy['inputs'] = [];
 
  // Assume package will only ever ship one package policy template for now
- const packagePolicyTemplate: RegistryConfigTemplate | null =
- packageInfo.config_templates && packageInfo.config_templates[0]
- ? packageInfo.config_templates[0]
+ const packagePolicyTemplate: RegistryPolicyTemplate | null =
+ packageInfo.policy_templates && packageInfo.policy_templates[0]
+ ? packageInfo.policy_templates[0]
  : null;
 
  // Create package policy input property

diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts
@@ -67,8 +67,8 @@ export interface RegistryPackage {
  assets?: string[];
  internal?: boolean;
  format_version: string;
- datasets?: Dataset[];
- config_templates?: RegistryConfigTemplate[];
+ data_streams?: RegistryDataStream[];
+ policy_templates?: RegistryPolicyTemplate[];
  download: string;
  path: string;
 }
@@ -80,7 +80,7 @@ interface RegistryImage {
  size?: string;
  type?: string;
 }
-export interface RegistryConfigTemplate {
+export interface RegistryPolicyTemplate {
  name: string;
  title: string;
  description: string;
@@ -127,8 +127,8 @@ export type RegistrySearchResult = Pick<
  | 'internal'
  | 'download'
  | 'path'
- | 'datasets'
- | 'config_templates'
+ | 'data_streams'
+ | 'policy_templates'
 >;
 
 export type ScreenshotItem = RegistryImage;
@@ -174,9 +174,9 @@ export type ElasticsearchAssetTypeToParts = Record<
  ElasticsearchAssetParts[]
 >;
 
-export interface Dataset {
+export interface RegistryDataStream {
  type: string;
- name: string;
+ dataset: string;
  title: string;
  release: string;
  streams?: RegistryStream[];

diff --git a/x-pack/plugins/ingest_manager/dev_docs/epm.md b/x-pack/plugins/ingest_manager/dev_docs/epm.md
@@ -26,5 +26,5 @@ When a package is installed or upgraded, certain Kibana and Elasticsearch assets
 ### Generation
 
 - Index templates are generated from `YAML` files contained in the package.
-- There is one index template per dataset.
-- For the generation of an index template, all `yml` files contained in the package subdirectory `dataset/DATASET_NAME/fields/` are used.
+- There is one index template per data stream.
+- For the generation of an index template, all `yml` files contained in the package subdirectory `data_stream/DATASET_NAME/fields/` are used.
diff --git a/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md b/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md
@@ -6,48 +6,48 @@ Overall documentation of Ingest Management is now maintained in the `elastic/sta
 Ingest Management enforces an indexing strategy to allow the system to automatically detect indices and run queries on it. In short the indexing strategy looks as following:
 
 ```
-{dataset.type}-{dataset.name}-{dataset.namespace}
+{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}
 ```
 
-The `{dataset.type}` can be `logs` or `metrics`. The `{dataset.namespace}` is the part where the user can use free form. The only two requirement are that it has only characters allowed in an Elasticsearch index name and does NOT contain a `-`. The `dataset` is defined by the data that is indexed. The same requirements as for the namespace apply. It is expected that the fields for type, namespace and dataset are part of each event and are constant keywords. If there is a dataset or a namespace with a `-` inside, it is recommended to replace it either by a `.` or a `_`.
+The `{data_stream.type}` can be `logs` or `metrics`. The `{data_stream.namespace}` is the part where the user can use free form. The only two requirement are that it has only characters allowed in an Elasticsearch index name and does NOT contain a `-`. The `data_stream` is defined by the data that is indexed. The same requirements as for the namespace apply. It is expected that the fields for type, dataset, and namespace are part of each event and are constant keywords. If there is a dataset or a namespace with a `-` inside, it is recommended to replace it either by a `.` or a `_`.
 
-Note: More `{dataset.type}`s might be added in the future like `traces`.
+Note: More `{data_stream.type}`s might be added in the future like `traces`.
 
 This indexing strategy has a few advantages:
 
-* Each index contains only the fields which are relevant for the dataset. This leads to more dense indices and better field completion.
-* ILM policies can be applied per namespace per dataset.
-* Rollups can be specified per namespace per dataset.
-* Having the namespace user configurable makes setting security permissions possible.
-* Having a global metrics and logs template, allows to create new indices on demand which still follow the convention. This is common in the case of k8s as an example.
-* Constant keywords allow to narrow down the indices we need to access for querying very efficiently. This is especially relevant in environments which a large number of indices or with indices on slower nodes.
+- Each index contains only the fields which are relevant for the datta stream. This leads to more dense indices and better field completion.
+- ILM policies can be applied per namespace per data stream.
+- Rollups can be specified per namespace per data stream.
+- Having the namespace user configurable makes setting security permissions possible.
+- Having a global metrics and logs template, allows to create new indices on demand which still follow the convention. This is common in the case of k8s as an example.
+- Constant keywords allow to narrow down the indices we need to access for querying very efficiently. This is especially relevant in environments which a large number of indices or with indices on slower nodes.
 
 Overall it creates smaller indices in size, makes querying more efficient and allows users to define their own naming parts in namespace and still benefiting from all features that can be built on top of the indexing startegy.
 
 ## Ingest Pipeline
 
-The ingest pipelines for a specific dataset will have the following naming scheme:
+The ingest pipelines for a specific data stream will have the following naming scheme:
 
 ```
-{dataset.type}-{dataset.name}-{package.version}
+{data_stream.type}-{data_stream.dataset}-{package.version}
 ```
 
-As an example, the ingest pipeline for the Nginx access logs is called `logs-nginx.access-3.4.1`. The same ingest pipeline is used for all namespaces. It is possible that a dataset has multiple ingest pipelines in which case a suffix is added to the name.
+As an example, the ingest pipeline for the Nginx access logs is called `logs-nginx.access-3.4.1`. The same ingest pipeline is used for all namespaces. It is possible that a data stream has multiple ingest pipelines in which case a suffix is added to the name.
 
 The version is included in each pipeline to allow upgrades. The pipeline itself is listed in the index template and is automatically applied at ingest time.
 
 ## Templates & ILM Policies
 
-To make the above strategy possible, alias templates are required. For each type there is a basic alias template with a default ILM policy. These default templates apply to all indices which follow the indexing strategy and do not have a more specific dataset alias template.
+To make the above strategy possible, alias templates are required. For each type there is a basic alias template with a default ILM policy. These default templates apply to all indices which follow the indexing strategy and do not have a more specific data stream alias template.
 
 The `metrics` and `logs` alias template contain all the basic fields from ECS.
 
 Each type template contains an ILM policy. Modifying this default ILM policy will affect all data covered by the default templates.
 
-The templates for a dataset are called as following:
+The templates for a data stream are called as following:
 
 ```
-{dataset.type}-{dataset.name}
+{data_stream.type}-{data_stream.dataset}
 ```
 
 The pattern used inside the index template is `{type}-{dataset}-*` to match all namespaces.