[APM] Syncs agent config settings to APM Fleet policies (elastic#100744)

* [APM] Syncs agent config settings to APM Fleet policies (elastic#95501)

* fixes eslint issues

* fixes malformed line comment

* - consolidated logic that applies agent configurations to package policy objects
- update package policy agent_configs to include etag, agent.name, and change settings -> config

* Synchronizes agent configs whenever configuration is deleted.

* PR feedback

* nest agent_config within `apm-server` in the package policy input

* nests agent_config under the requried 'value' property of config['apm-server']
in order to pass validation checks

* - externalizes getApmPackagePolicies for reusability
- parallelizes operations for improved performance

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/apm/kibana.json

@@ -23,7 +23,8 @@
  "security",
  "ml",
  "home",
- "maps"
+ "maps",
+ "fleet"
  ],
  "server": true,
  "ui": true,

x-pack/plugins/apm/server/lib/fleet/get_apm_package_policies.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+ CoreSetup,
+ CoreStart,
+ SavedObjectsClientContract,
+} from 'kibana/server';
+import { APMPluginStartDependencies } from '../../types';
+import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client';
+
+export async function getApmPackgePolicies({
+ core,
+ fleetPluginStart,
+}: {
+ core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+ fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+}) {
+ // @ts-ignore
+ const savedObjectsClient: SavedObjectsClientContract = await getInternalSavedObjectsClient(
+ core.setup
+ );
+ return await fleetPluginStart.packagePolicyService.list(savedObjectsClient, {
+ kuery: 'ingest-package-policies.package.name:apm',
+ });
+}

x-pack/plugins/apm/server/lib/fleet/register_fleet_policy_callbacks.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { APMPlugin, APMRouteHandlerResources } from '../..';
+import { listConfigurations } from '../settings/agent_configuration/list_configurations';
+import { setupRequest } from '../helpers/setup_request';
+import { APMPluginStartDependencies } from '../../types';
+import { ExternalCallback } from '../../../../fleet/server';
+import { AGENT_NAME } from '../../../common/elasticsearch_fieldnames';
+import { AgentConfiguration } from '../../../common/agent_configuration/configuration_types';
+
+export async function registerFleetPolicyCallbacks({
+ plugins,
+ ruleDataClient,
+ config,
+ logger,
+}: {
+ plugins: APMRouteHandlerResources['plugins'];
+ ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
+ config: NonNullable<APMPlugin['currentConfig']>;
+ logger: NonNullable<APMPlugin['logger']>;
+}) {
+ if (!plugins.fleet) {
+ return;
+ }
+ const fleetPluginStart = await plugins.fleet.start();
+
+ // Registers a callback invoked when a policy is created to populate the APM
+ // integration policy with pre-existing agent configurations
+ registerAgentConfigExternalCallback({
+ fleetPluginStart,
+ callbackName: 'packagePolicyCreate',
+ plugins,
+ ruleDataClient,
+ config,
+ logger,
+ });
+
+ // Registers a callback invoked when a policy is updated to populate the APM
+ // integration policy with existing agent configurations
+ registerAgentConfigExternalCallback({
+ fleetPluginStart,
+ callbackName: 'packagePolicyUpdate',
+ plugins,
+ ruleDataClient,
+ config,
+ logger,
+ });
+}
+
+type ExternalCallbackParams = Parameters<ExternalCallback[1]>;
+type PackagePolicy = ExternalCallbackParams[0];
+type Context = ExternalCallbackParams[1];
+type Request = ExternalCallbackParams[2];
+
+function registerAgentConfigExternalCallback({
+ fleetPluginStart,
+ callbackName,
+ plugins,
+ ruleDataClient,
+ config,
+ logger,
+}: {
+ fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+ callbackName: ExternalCallback[0];
+ plugins: APMRouteHandlerResources['plugins'];
+ ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
+ config: NonNullable<APMPlugin['currentConfig']>;
+ logger: NonNullable<APMPlugin['logger']>;
+}) {
+ const callbackFn: ExternalCallback[1] = async (
+ packagePolicy: PackagePolicy,
+ context: Context,
+ request: Request
+ ) => {
+ if (packagePolicy.package?.name !== 'apm') {
+ return packagePolicy;
+ }
+ const setup = await setupRequest({
+ context: context as any,
+ params: { query: { _inspect: false } },
+ core: null as any,
+ plugins,
+ request,
+ config,
+ logger,
+ ruleDataClient,
+ });
+ const agentConfigurations = await listConfigurations({ setup });
+ return getPackagePolicyWithAgentConfigurations(
+ packagePolicy,
+ agentConfigurations
+ );
+ };
+
+ fleetPluginStart.registerExternalCallback(callbackName, callbackFn);
+}
+
+const APM_SERVER = 'apm-server';
+
+// Immutable function applies the given package policy with a set of agent configurations
+export function getPackagePolicyWithAgentConfigurations(
+ packagePolicy: PackagePolicy,
+ agentConfigurations: AgentConfiguration[]
+) {
+ const [firstInput, ...restInputs] = packagePolicy.inputs;
+ const apmServerValue = firstInput?.config?.[APM_SERVER].value;
+ return {
+ ...packagePolicy,
+ inputs: [
+ {
+ ...firstInput,
+ config: {
+ [APM_SERVER]: {
+ value: {
+ ...apmServerValue,
+ agent_config: agentConfigurations.map((configuration) => ({
+ service: configuration.service,
+ config: configuration.settings,
+ etag: configuration.etag,
+ [AGENT_NAME]: configuration.agent_name,
+ })),
+ },
+ },
+ },
+ },
+ ...restInputs,
+ ],
+ };
+}

x-pack/plugins/apm/server/lib/fleet/sync_agent_configs_to_apm_package_policies.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+ CoreSetup,
+ CoreStart,
+ SavedObjectsClientContract,
+} from 'kibana/server';
+import { APMPluginStartDependencies } from '../../types';
+import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client';
+import { Setup } from '../helpers/setup_request';
+import { listConfigurations } from '../settings/agent_configuration/list_configurations';
+import { getApmPackgePolicies } from './get_apm_package_policies';
+import { getPackagePolicyWithAgentConfigurations } from './register_fleet_policy_callbacks';
+
+export async function syncAgentConfigsToApmPackagePolicies({
+ core,
+ fleetPluginStart,
+ setup,
+}: {
+ core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+ fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
+ setup: Setup;
+}) {
+ const coreStart = await core.start();
+ const esClient = coreStart.elasticsearch.client.asInternalUser;
+ const [
+ savedObjectsClient,
+ agentConfigurations,
+ packagePolicies,
+ ] = await Promise.all([
+ getInternalSavedObjectsClient(core.setup),
+ listConfigurations({ setup }),
+ getApmPackgePolicies({
+ core,
+ fleetPluginStart,
+ }),
+ ]);
+
+ return Promise.all(
+ packagePolicies.items.map(async (item) => {
+ const { id, revision, updated_at, updated_by, ...packagePolicy } = item; // eslint-disable-line @typescript-eslint/naming-convention
+ const updatedPackagePolicy = getPackagePolicyWithAgentConfigurations(
+ packagePolicy,
+ agentConfigurations
+ );
+ return fleetPluginStart.packagePolicyService.update(
+ (savedObjectsClient as unknown) as SavedObjectsClientContract,
+ esClient,
+ id,
+ updatedPackagePolicy
+ );
+ })
+ );
+}

x-pack/plugins/apm/server/plugin.ts

@@ -24,6 +24,7 @@ import { mergeConfigs } from './index';
 import { UI_SETTINGS } from '../../../../src/plugins/data/common';
 import { APM_FEATURE, registerFeaturesUsage } from './feature';
 import { registerApmAlerts } from './lib/alerts/register_apm_alerts';
+import { registerFleetPolicyCallbacks } from './lib/fleet/register_fleet_policy_callbacks';
 import { createApmTelemetry } from './lib/apm_telemetry';
 import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client';
 import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client';
@@ -186,6 +187,19 @@ export class APMPlugin
  ready,
  });
 
+ const resourcePlugins = mapValues(plugins, (value, key) => {
+ return {
+ setup: value,
+ start: () =>
+ core.getStartServices().then((services) => {
+ const [, pluginsStartContracts] = services;
+ return pluginsStartContracts[
+ key as keyof APMPluginStartDependencies
+ ];
+ }),
+ };
+ }) as APMRouteHandlerResources['plugins'];
+
  registerRoutes({
  core: {
  setup: core,
@@ -195,18 +209,7 @@ export class APMPlugin
  config: currentConfig,
  repository: getGlobalApmServerRouteRepository(),
  ruleDataClient,
- plugins: mapValues(plugins, (value, key) => {
- return {
- setup: value,
- start: () =>
- core.getStartServices().then((services) => {
- const [, pluginsStartContracts] = services;
- return pluginsStartContracts[
- key as keyof APMPluginStartDependencies
- ];
- }),
- };
- }) as APMRouteHandlerResources['plugins'],
+ plugins: resourcePlugins,
  });
 
  const boundGetApmIndices = async () =>
@@ -225,6 +228,13 @@ export class APMPlugin
  });
  }
 
+ registerFleetPolicyCallbacks({
+ plugins: resourcePlugins,
+ ruleDataClient,
+ config: this.currentConfig,
+ logger: this.logger,
+ });
+
  return {
  config$: mergedConfig$,
  getApmIndices: boundGetApmIndices,

x-pack/plugins/apm/server/routes/settings/agent_configuration.ts

@@ -25,6 +25,7 @@ import {
 } from '../../../common/agent_configuration/runtime_types/agent_configuration_intake_rt';
 import { getSearchAggregatedTransactions } from '../../lib/helpers/aggregated_transactions';
 import { createApmServerRouteRepository } from '../create_apm_server_route_repository';
+import { syncAgentConfigsToApmPackagePolicies } from '../../lib/fleet/sync_agent_configs_to_apm_package_policies';
 
 // get list of configurations
 const agentConfigurationRoute = createApmServerRoute({
@@ -78,7 +79,7 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
  }),
  handler: async (resources) => {
  const setup = await setupRequest(resources);
- const { params, logger } = resources;
+ const { params, logger, core } = resources;
 
  const { service } = params.body;
 
@@ -95,10 +96,23 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
  `Deleting config ${service.name}/${service.environment} (${config._id})`
  );
 
- return await deleteConfiguration({
+ const deleteConfigurationResult = await deleteConfiguration({
  configurationId: config._id,
  setup,
  });
+
+ if (resources.plugins.fleet) {
+ await syncAgentConfigsToApmPackagePolicies({
+ core,
+ fleetPluginStart: await resources.plugins.fleet.start(),
+ setup,
+ });
+ logger.info(
+ `Updated Fleet integration policy for APM to remove the deleted agent configuration.`
+ );
+ }
+
+ return deleteConfigurationResult;
  },
 });
 
@@ -114,7 +128,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
  ]),
  handler: async (resources) => {
  const setup = await setupRequest(resources);
- const { params, logger } = resources;
+ const { params, logger, core } = resources;
  const { body, query } = params;
 
  // if the config already exists, it is fetched and updated
@@ -142,6 +156,17 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
  configurationIntake: body,
  setup,
  });
+
+ if (resources.plugins.fleet) {
+ await syncAgentConfigsToApmPackagePolicies({
+ core,
+ fleetPluginStart: await resources.plugins.fleet.start(),
+ setup,
+ });
+ logger.info(
+ `Saved latest agent settings to Fleet integration policy for APM.`
+ );
+ }
  },
 });
 

x-pack/plugins/apm/server/types.ts

@@ -43,6 +43,10 @@ import {
  TaskManagerSetupContract,
  TaskManagerStartContract,
 } from '../../task_manager/server';
+import {
+ FleetSetupContract as FleetPluginSetup,
+ FleetStartContract as FleetPluginStart,
+} from '../../fleet/server';
 import { APMConfig } from '.';
 import { getApmIndices } from './lib/settings/apm_indices/get_apm_indices';
 import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client';
@@ -123,6 +127,10 @@ interface DependencyMap {
  setup: RuleRegistryPluginSetupContract;
  start: RuleRegistryPluginStartContract;
  };
+ fleet: {
+ setup: FleetPluginSetup;
+ start: FleetPluginStart;
+ };
 }
 
 const requiredDependencies = [
@@ -148,6 +156,7 @@ const optionalDependencies = [
  'ml',
  'home',
  'maps',
+ 'fleet',
 ] as const;
 
 type RequiredDependencies = Pick<