[Ingest Manager] Support limiting integrations on an agent config (#7…

…0542) (#70895)

* Add API endpoint and hook for retrieving restricted packages

* Filter out restricted packages already in use from list of integrations available for an agent config

* Allow list agent configs to optionally return expanded package configs, re

* Filter out agent configs which already use the restricted package already from list of agent configs available for an integration

* Allow more than 20 agent configs to be shown

* Rename restricted to limited; add some common methods to DRY

* Add limited package check on server side

* Adjust copy wording

* Fix typings

* Add some package config api integration tests, update es archive mappings

* Move test to dockerized integation tests directory; move existing epm tests to their own directory

* Remove extra assignPackageConfigs() - already handled in packageConfigService.create()

* Review fixes

* Fix type, reenabled skipped test

* Move new EPM integration test file

x-pack/plugins/ingest_manager/common/constants/routes.ts

@@ -17,6 +17,7 @@ const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`;
 const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`;
 export const EPM_API_ROUTES = {
  LIST_PATTERN: EPM_PACKAGES_MANY,
+ LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`,
  INFO_PATTERN: EPM_PACKAGES_ONE,
  INSTALL_PATTERN: EPM_PACKAGES_ONE,
  DELETE_PATTERN: EPM_PACKAGES_ONE,

x-pack/plugins/ingest_manager/common/services/index.ts

@@ -3,11 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import * as AgentStatusKueryHelper from './agent_status';
-
 export * from './routes';
+export * as AgentStatusKueryHelper from './agent_status';
 export { packageToPackageConfigInputs, packageToPackageConfig } from './package_to_config';
 export { storedPackageConfigsToAgentInputs } from './package_configs_to_agent_inputs';
 export { configToYaml } from './config_to_yaml';
-export { AgentStatusKueryHelper };
+export { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from './limited_package';
 export { decodeCloudId } from './decode_cloud_id';

x-pack/plugins/ingest_manager/common/services/limited_package.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { PackageInfo, AgentConfig, PackageConfig } from '../types';
+
+// Assume packages only ever include 1 config template for now
+export const isPackageLimited = (packageInfo: PackageInfo): boolean => {
+ return packageInfo.config_templates?.[0]?.multiple === false;
+};
+
+export const doesAgentConfigAlreadyIncludePackage = (
+ agentConfig: AgentConfig,
+ packageName: string
+): boolean => {
+ if (agentConfig.package_configs.length && typeof agentConfig.package_configs[0] === 'string') {
+ throw new Error('Unable to read full package config information');
+ }
+ return (agentConfig.package_configs as PackageConfig[])
+ .map((packageConfig) => packageConfig.package?.name || '')
+ .includes(packageName);
+};

x-pack/plugins/ingest_manager/common/services/routes.ts

@@ -27,6 +27,10 @@ export const epmRouteService = {
  return EPM_API_ROUTES.LIST_PATTERN;
  },
 
+ getListLimitedPath: () => {
+ return EPM_API_ROUTES.LIMITED_LIST_PATTERN;
+ },
+
  getInfoPath: (pkgkey: string) => {
  return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey);
  },

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -79,6 +79,7 @@ export interface RegistryConfigTemplate {
  title: string;
  description: string;
  inputs: RegistryInput[];
+ multiple?: boolean;
 }
 
 export interface RegistryInput {

x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts

@@ -7,7 +7,9 @@ import { AgentConfig, NewAgentConfig, FullAgentConfig } from '../models';
 import { ListWithKuery } from './common';
 
 export interface GetAgentConfigsRequest {
- query: ListWithKuery;
+ query: ListWithKuery & {
+ full?: boolean;
+ };
 }
 
 export type GetAgentConfigsResponseItem = AgentConfig & { agents?: number };

x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts

@@ -3,8 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { HttpFetchQuery } from 'src/core/public';
 
-export interface ListWithKuery {
+export interface ListWithKuery extends HttpFetchQuery {
  page?: number;
  perPage?: number;
  sortField?: string;

x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts

@@ -34,6 +34,11 @@ export interface GetPackagesResponse {
  success: boolean;
 }
 
+export interface GetLimitedPackagesResponse {
+ response: string[];
+ success: boolean;
+}
+
 export interface GetFileRequest {
  params: {
  pkgkey: string;

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts

@@ -3,7 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { HttpFetchQuery } from 'src/core/public';
 import {
  useRequest,
  sendRequest,
@@ -12,6 +11,7 @@ import {
 } from './use_request';
 import { agentConfigRouteService } from '../../services';
 import {
+ GetAgentConfigsRequest,
  GetAgentConfigsResponse,
  GetOneAgentConfigResponse,
  GetFullAgentConfigResponse,
@@ -25,7 +25,7 @@ import {
  DeleteAgentConfigResponse,
 } from '../../types';
 
-export const useGetAgentConfigs = (query: HttpFetchQuery = {}) => {
+export const useGetAgentConfigs = (query?: GetAgentConfigsRequest['query']) => {
  return useRequest<GetAgentConfigsResponse>({
  path: agentConfigRouteService.getListPath(),
  method: 'get',

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts

@@ -10,6 +10,7 @@ import { epmRouteService } from '../../services';
 import {
  GetCategoriesResponse,
  GetPackagesResponse,
+ GetLimitedPackagesResponse,
  GetInfoResponse,
  InstallPackageResponse,
  DeletePackageResponse,
@@ -30,6 +31,13 @@ export const useGetPackages = (query: HttpFetchQuery = {}) => {
  });
 };
 
+export const useGetLimitedPackages = () => {
+ return useRequest<GetLimitedPackagesResponse>({
+ path: epmRouteService.getListLimitedPath(),
+ method: 'get',
+ });
+};
+
 export const useGetPackageInfoByKey = (pkgkey: string) => {
  return useRequest<GetInfoResponse>({
  path: epmRouteService.getInfoPath(pkgkey),

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx

@@ -9,6 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer, EuiTextColor } from '@elastic/eui';
 import { Error } from '../../../components';
 import { AgentConfig, PackageInfo, GetAgentConfigsResponseItem } from '../../../types';
+import { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from '../../../services';
 import { useGetPackageInfoByKey, useGetAgentConfigs, sendGetOneAgentConfig } from '../../../hooks';
 
 export const StepSelectConfig: React.FunctionComponent<{
@@ -24,7 +25,12 @@ export const StepSelectConfig: React.FunctionComponent<{
  const [selectedConfigError, setSelectedConfigError] = useState<Error>();
 
  // Fetch package info
- const { data: packageInfoData, error: packageInfoError } = useGetPackageInfoByKey(pkgkey);
+ const {
+ data: packageInfoData,
+ error: packageInfoError,
+ isLoading: packageInfoLoading,
+ } = useGetPackageInfoByKey(pkgkey);
+ const isLimitedPackage = (packageInfoData && isPackageLimited(packageInfoData.response)) || false;
 
  // Fetch agent configs info
  const {
@@ -36,6 +42,7 @@ export const StepSelectConfig: React.FunctionComponent<{
  perPage: 1000,
  sortField: 'name',
  sortOrder: 'asc',
+ full: true,
  });
  const agentConfigs = agentConfigsData?.items || [];
  const agentConfigsById = agentConfigs.reduce(
@@ -112,12 +119,18 @@ export const StepSelectConfig: React.FunctionComponent<{
  searchable
  allowExclusions={false}
  singleSelection={true}
- isLoading={isAgentConfigsLoading}
- options={agentConfigs.map(({ id, name, description }) => {
+ isLoading={isAgentConfigsLoading || packageInfoLoading}
+ options={agentConfigs.map((agentConf) => {
+ const alreadyHasLimitedPackage =
+ (isLimitedPackage &&
+ packageInfoData &&
+ doesAgentConfigAlreadyIncludePackage(agentConf, packageInfoData.response.name)) ||
+ false;
  return {
- label: name,
- key: id,
- checked: selectedConfigId === id ? 'on' : undefined,
+ label: agentConf.name,
+ key: agentConf.id,
+ checked: selectedConfigId === agentConf.id ? 'on' : undefined,
+ disabled: alreadyHasLimitedPackage,
  'data-test-subj': 'agentConfigItem',
  };
  })}

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx

@@ -8,8 +8,13 @@ import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer } from '@elastic/eui';
 import { Error } from '../../../components';
-import { AgentConfig, PackageInfo } from '../../../types';
-import { useGetOneAgentConfig, useGetPackages, sendGetPackageInfoByKey } from '../../../hooks';
+import { AgentConfig, PackageInfo, PackageConfig, GetPackagesResponse } from '../../../types';
+import {
+ useGetOneAgentConfig,
+ useGetPackages,
+ useGetLimitedPackages,
+ sendGetPackageInfoByKey,
+} from '../../../hooks';
 import { PackageIcon } from '../../../components/package_icon';
 
 export const StepSelectPackage: React.FunctionComponent<{
@@ -28,12 +33,27 @@ export const StepSelectPackage: React.FunctionComponent<{
  const { data: agentConfigData, error: agentConfigError } = useGetOneAgentConfig(agentConfigId);
 
  // Fetch packages info
+ // Filter out limited packages already part of selected agent config
+ const [packages, setPackages] = useState<GetPackagesResponse['response']>([]);
  const {
  data: packagesData,
  error: packagesError,
  isLoading: isPackagesLoading,
  } = useGetPackages();
- const packages = packagesData?.response || [];
+ const {
+ data: limitedPackagesData,
+ isLoading: isLimitedPackagesLoading,
+ } = useGetLimitedPackages();
+ useEffect(() => {
+ if (packagesData?.response && limitedPackagesData?.response && agentConfigData?.item) {
+ const allPackages = packagesData.response;
+ const limitedPackages = limitedPackagesData.response;
+ const usedLimitedPackages = (agentConfigData.item.package_configs as PackageConfig[])
+ .map((packageConfig) => packageConfig.package?.name || '')
+ .filter((pkgName) => limitedPackages.includes(pkgName));
+ setPackages(allPackages.filter((pkg) => !usedLimitedPackages.includes(pkg.name)));
+ }
+ }, [packagesData, limitedPackagesData, agentConfigData]);
 
  // Update parent agent config state
  useEffect(() => {
@@ -101,7 +121,7 @@ export const StepSelectPackage: React.FunctionComponent<{
  searchable
  allowExclusions={false}
  singleSelection={true}
- isLoading={isPackagesLoading}
+ isLoading={isPackagesLoading || isLimitedPackagesLoading}
  options={packages.map(({ title, name, version, icons }) => {
  const pkgkey = `${name}-${version}`;
  return {

x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts

@@ -7,6 +7,7 @@
 export { getFlattenedObject } from '../../../../../../../src/core/public';
 
 export {
+ AgentStatusKueryHelper,
  agentConfigRouteService,
  packageConfigRouteService,
  dataStreamRouteService,
@@ -21,5 +22,6 @@ export {
  packageToPackageConfigInputs,
  storedPackageConfigsToAgentInputs,
  configToYaml,
- AgentStatusKueryHelper,
+ isPackageLimited,
+ doesAgentConfigAlreadyIncludePackage,
 } from '../../../../common';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts

@@ -24,6 +24,7 @@ export {
  // API schema - misc setup, status
  GetFleetStatusResponse,
  // API schemas - Agent Config
+ GetAgentConfigsRequest,
  GetAgentConfigsResponse,
  GetAgentConfigsResponseItem,
  GetOneAgentConfigResponse,
@@ -92,6 +93,7 @@ export {
  ServiceName,
  GetCategoriesResponse,
  GetPackagesResponse,
+ GetLimitedPackagesResponse,
  GetInfoResponse,
  InstallPackageResponse,
  DeletePackageResponse,

x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts

@@ -38,8 +38,12 @@ export const getAgentConfigsHandler: RequestHandler<
  TypeOf<typeof GetAgentConfigsRequestSchema.query>
 > = async (context, request, response) => {
  const soClient = context.core.savedObjects.client;
+ const { full: withPackageConfigs = false, ...restOfQuery } = request.query;
  try {
- const { items, total, page, perPage } = await agentConfigService.list(soClient, request.query);
+ const { items, total, page, perPage } = await agentConfigService.list(soClient, {
+ withPackageConfigs,
+ ...restOfQuery,
+ });
  const body: GetAgentConfigsResponse = {
  items,
  total,
@@ -103,6 +107,7 @@ export const createAgentConfigHandler: RequestHandler<
  TypeOf<typeof CreateAgentConfigRequestSchema.body>
 > = async (context, request, response) => {
  const soClient = context.core.savedObjects.client;
+ const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
  const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
  const withSysMonitoring = request.query.sys_monitoring ?? false;
  try {
@@ -128,15 +133,9 @@ export const createAgentConfigHandler: RequestHandler<
  if (withSysMonitoring && newSysPackageConfig !== undefined && agentConfig !== undefined) {
  newSysPackageConfig.config_id = agentConfig.id;
  newSysPackageConfig.namespace = agentConfig.namespace;
- const sysPackageConfig = await packageConfigService.create(soClient, newSysPackageConfig, {
+ await packageConfigService.create(soClient, callCluster, newSysPackageConfig, {
  user,
  });
-
- if (sysPackageConfig) {
- agentConfig = await agentConfigService.assignPackageConfigs(soClient, agentConfig.id, [
- sysPackageConfig.id,
- ]);
- }
  }
 
  const body: CreateAgentConfigResponse = {

x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts

@@ -5,27 +5,29 @@
  */
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server';
-import {
- GetPackagesRequestSchema,
- GetFileRequestSchema,
- GetInfoRequestSchema,
- InstallPackageRequestSchema,
- DeletePackageRequestSchema,
-} from '../../types';
 import {
  GetInfoResponse,
  InstallPackageResponse,
  DeletePackageResponse,
  GetCategoriesResponse,
  GetPackagesResponse,
+ GetLimitedPackagesResponse,
 } from '../../../common';
+import {
+ GetPackagesRequestSchema,
+ GetFileRequestSchema,
+ GetInfoRequestSchema,
+ InstallPackageRequestSchema,
+ DeletePackageRequestSchema,
+} from '../../types';
 import {
  getCategories,
  getPackages,
  getFile,
  getPackageInfo,
  installPackage,
  removeInstallation,
+ getLimitedPackages,
 } from '../../services/epm/packages';
 
 export const getCategoriesHandler: RequestHandler = async (context, request, response) => {
@@ -69,6 +71,25 @@ export const getListHandler: RequestHandler<
  }
 };
 
+export const getLimitedListHandler: RequestHandler = async (context, request, response) => {
+ try {
+ const savedObjectsClient = context.core.savedObjects.client;
+ const res = await getLimitedPackages({ savedObjectsClient });
+ const body: GetLimitedPackagesResponse = {
+ response: res,
+ success: true,
+ };
+ return response.ok({
+ body,
+ });
+ } catch (e) {
+ return response.customError({
+ statusCode: 500,
+ body: { message: e.message },
+ });
+ }
+};
+
 export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.params>> = async (
  context,
  request,