diff --git a/x-pack/plugins/ingest_manager/common/mocks.ts b/x-pack/plugins/ingest_manager/common/mocks.ts new file mode 100644 index 00000000000000..131917af445952 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/mocks.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { NewPackageConfig, PackageConfig } from './types/models/package_config'; + +export const createNewPackageConfigMock = () => { + return { + name: 'endpoint-1', + description: '', + namespace: 'default', + enabled: true, + config_id: '93c46720-c217-11ea-9906-b5b8a21b268e', + output_id: '', + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.9.0', + }, + inputs: [], + } as NewPackageConfig; +}; + +export const createPackageConfigMock = () => { + const newPackageConfig = createNewPackageConfigMock(); + return { + ...newPackageConfig, + id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', + version: 'abcd', + revision: 1, + updated_at: '2020-06-25T16:03:38.159292', + updated_by: 'kibana', + created_at: '2020-06-25T16:03:38.159292', + created_by: 'kibana', + inputs: [ + { + config: {}, + }, + ], + } as PackageConfig; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts new file mode 100644 index 00000000000000..bb035a19f33d60 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { loggerMock } from 'src/core/server/logging/logger.mock'; +import { createNewPackageConfigMock } from '../../../ingest_manager/common/mocks'; +import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; +import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { getPackageConfigCreateCallback } from './ingest_integration'; + +describe('ingest_integration tests ', () => { + describe('ingest_integration sanity checks', () => { + test('policy is updated with manifest', async () => { + const logger = loggerMock.create(); + const manifestManager = getManifestManagerMock(); + const callback = getPackageConfigCreateCallback(logger, manifestManager); + const policyConfig = createNewPackageConfigMock(); + const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); + expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); + expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({ + artifacts: { + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', + decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + decoded_size: 287, + encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', + encoded_size: 133, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + }, + }, + manifest_version: 'WzAsMF0=', + schema_version: 'v1', + }); + }); + + test('policy is returned even if error is encountered during artifact sync', async () => { + const logger = loggerMock.create(); + const manifestManager = getManifestManagerMock(); + manifestManager.syncArtifacts = jest.fn().mockRejectedValue([new Error('error updating')]); + const lastDispatched = await manifestManager.getLastDispatchedManifest(); + const callback = getPackageConfigCreateCallback(logger, manifestManager); + const policyConfig = createNewPackageConfigMock(); + const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); + expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); + expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( + lastDispatched.toEndpointFormat() + ); + }); + + test('initial policy creation succeeds if snapshot retrieval fails', async () => { + const logger = loggerMock.create(); + const manifestManager = getManifestManagerMock(); + const lastDispatched = await manifestManager.getLastDispatchedManifest(); + manifestManager.getSnapshot = jest.fn().mockResolvedValue(null); + const callback = getPackageConfigCreateCallback(logger, manifestManager); + const policyConfig = createNewPackageConfigMock(); + const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); + expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); + expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( + lastDispatched.toEndpointFormat() + ); + }); + + test('subsequent policy creations succeed', async () => { + const logger = loggerMock.create(); + const manifestManager = getManifestManagerMock(); + const snapshot = await manifestManager.getSnapshot(); + manifestManager.getLastDispatchedManifest = jest.fn().mockResolvedValue(snapshot!.manifest); + manifestManager.getSnapshot = jest.fn().mockResolvedValue({ + manifest: snapshot!.manifest, + diffs: [], + }); + const callback = getPackageConfigCreateCallback(logger, manifestManager); + const policyConfig = createNewPackageConfigMock(); + const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); + expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); + expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( + snapshot!.manifest.toEndpointFormat() + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 1acec1e7c53ac7..e2522ac4af7786 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -8,7 +8,9 @@ import { Logger } from '../../../../../src/core/server'; import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; -import { ManifestManager } from './services/artifacts'; +import { ManifestManager, ManifestSnapshot } from './services/artifacts'; +import { reportErrors, ManifestConstants } from './lib/artifacts/common'; +import { ManifestSchemaVersion } from '../../common/endpoint/schema/common'; /** * Callback to handle creation of PackageConfigs in Ingest Manager @@ -29,58 +31,83 @@ export const getPackageConfigCreateCallback = ( // follow the types/schema expected let updatedPackageConfig = newPackageConfig as NewPolicyData; - // get snapshot based on exception-list-agnostic SOs - // with diffs from last dispatched manifest, if it exists - const snapshot = await manifestManager.getSnapshot({ initialize: true }); + // get current manifest from SO (last dispatched) + const manifest = ( + await manifestManager.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION) + )?.toEndpointFormat() ?? { + manifest_version: 'default', + schema_version: ManifestConstants.SCHEMA_VERSION as ManifestSchemaVersion, + artifacts: {}, + }; - if (snapshot === null) { - logger.warn('No manifest snapshot available.'); - return updatedPackageConfig; + // Until we get the Default Policy Configuration in the Endpoint package, + // we will add it here manually at creation time. + if (newPackageConfig.inputs.length === 0) { + updatedPackageConfig = { + ...newPackageConfig, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: { + value: manifest, + }, + policy: { + value: policyConfigFactory(), + }, + }, + }, + ], + }; } - if (snapshot.diffs.length > 0) { - // create new artifacts - await manifestManager.syncArtifacts(snapshot, 'add'); + let snapshot: ManifestSnapshot | null = null; + let success = true; + try { + // Try to get most up-to-date manifest data. - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - // @ts-ignore - if (newPackageConfig.inputs.length === 0) { - updatedPackageConfig = { - ...newPackageConfig, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - artifact_manifest: { - value: snapshot.manifest.toEndpointFormat(), - }, - policy: { - value: policyConfigFactory(), - }, - }, - }, - ], + // get snapshot based on exception-list-agnostic SOs + // with diffs from last dispatched manifest, if it exists + snapshot = await manifestManager.getSnapshot({ initialize: true }); + + if (snapshot && snapshot.diffs.length) { + // create new artifacts + const errors = await manifestManager.syncArtifacts(snapshot, 'add'); + if (errors.length) { + reportErrors(logger, errors); + throw new Error('Error writing new artifacts.'); + } + } + + if (snapshot) { + updatedPackageConfig.inputs[0].config.artifact_manifest = { + value: snapshot.manifest.toEndpointFormat(), }; } - } - try { + return updatedPackageConfig; + } catch (err) { + success = false; + logger.error(err); return updatedPackageConfig; } finally { - if (snapshot.diffs.length > 0) { - // TODO: let's revisit the way this callback happens... use promises? - // only commit when we know the package config was created + if (success && snapshot !== null) { try { - await manifestManager.commit(snapshot.manifest); + if (snapshot.diffs.length > 0) { + // TODO: let's revisit the way this callback happens... use promises? + // only commit when we know the package config was created + await manifestManager.commit(snapshot.manifest); - // clean up old artifacts - await manifestManager.syncArtifacts(snapshot, 'delete'); + // clean up old artifacts + await manifestManager.syncArtifacts(snapshot, 'delete'); + } } catch (err) { logger.error(err); } + } else if (snapshot === null) { + logger.error('No manifest snapshot available.'); } } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 9ad4554b302036..71d14eb1226d57 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'src/core/server'; export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', @@ -16,3 +17,9 @@ export const ManifestConstants = { SCHEMA_VERSION: 'v1', INITIAL_VERSION: 'WzAsMF0=', }; + +export const reportErrors = (logger: Logger, errors: Error[]) => { + errors.forEach((err) => { + logger.error(err); + }); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index aa7f56e815d58f..583f4499f591b5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -11,6 +11,7 @@ import { TaskManagerStartContract, } from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; +import { reportErrors } from './common'; export const ManifestTaskConstants = { TIMEOUT: '1m', @@ -88,19 +89,36 @@ export class ManifestTask { return; } + let errors: Error[] = []; try { // get snapshot based on exception-list-agnostic SOs // with diffs from last dispatched manifest const snapshot = await manifestManager.getSnapshot(); if (snapshot && snapshot.diffs.length > 0) { // create new artifacts - await manifestManager.syncArtifacts(snapshot, 'add'); + errors = await manifestManager.syncArtifacts(snapshot, 'add'); + if (errors.length) { + reportErrors(this.logger, errors); + throw new Error('Error writing new artifacts.'); + } // write to ingest-manager package config - await manifestManager.dispatch(snapshot.manifest); + errors = await manifestManager.dispatch(snapshot.manifest); + if (errors.length) { + reportErrors(this.logger, errors); + throw new Error('Error dispatching manifest.'); + } // commit latest manifest state to user-artifact-manifest SO - await manifestManager.commit(snapshot.manifest); + const error = await manifestManager.commit(snapshot.manifest); + if (error) { + reportErrors(this.logger, [error]); + throw new Error('Error committing manifest.'); + } // clean up old artifacts - await manifestManager.syncArtifacts(snapshot, 'delete'); + errors = await manifestManager.syncArtifacts(snapshot, 'delete'); + if (errors.length) { + reportErrors(this.logger, errors); + throw new Error('Error cleaning up outdated artifacts.'); + } } } catch (err) { this.logger.error(err); diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 55d7baec36dc6d..6a8c26e08d9dd9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -6,6 +6,8 @@ import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { loggerMock } from 'src/core/server/logging/logger.mock'; import { xpackMocks } from '../../../../mocks'; import { AgentService, @@ -63,8 +65,8 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< > => { return { agentService: createMockAgentService(), + logger: loggerMock.create(), savedObjectsStart: savedObjectsServiceMock.createStartContract(), - // @ts-ignore manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index b7f99fe6fe297c..ed97d04eecee65 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -7,38 +7,38 @@ import * as t from 'io-ts'; import { operator } from '../../../../../lists/common/schemas'; +export const translatedEntryMatchAnyMatcher = t.keyof({ + exact_cased_any: null, + exact_caseless_any: null, +}); +export type TranslatedEntryMatchAnyMatcher = t.TypeOf; + export const translatedEntryMatchAny = t.exact( t.type({ field: t.string, operator, - type: t.keyof({ - exact_cased_any: null, - exact_caseless_any: null, - }), + type: translatedEntryMatchAnyMatcher, value: t.array(t.string), }) ); export type TranslatedEntryMatchAny = t.TypeOf; -export const translatedEntryMatchAnyMatcher = translatedEntryMatchAny.type.props.type; -export type TranslatedEntryMatchAnyMatcher = t.TypeOf; +export const translatedEntryMatchMatcher = t.keyof({ + exact_cased: null, + exact_caseless: null, +}); +export type TranslatedEntryMatchMatcher = t.TypeOf; export const translatedEntryMatch = t.exact( t.type({ field: t.string, operator, - type: t.keyof({ - exact_cased: null, - exact_caseless: null, - }), + type: translatedEntryMatchMatcher, value: t.string, }) ); export type TranslatedEntryMatch = t.TypeOf; -export const translatedEntryMatchMatcher = translatedEntryMatch.type.props.type; -export type TranslatedEntryMatchMatcher = t.TypeOf; - export const translatedEntryMatcher = t.union([ translatedEntryMatchMatcher, translatedEntryMatchAnyMatcher, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index dfbe2572076d00..3bdc5dfbcbd45b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -4,9 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -// eslint-disable-next-line max-classes-per-file import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; import { Logger } from 'src/core/server'; +import { createPackageConfigMock } from '../../../../../../ingest_manager/common/mocks'; +import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; +import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { listMock } from '../../../../../../lists/server/mocks'; import { @@ -21,40 +23,6 @@ import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; import { ManifestManager } from './manifest_manager'; -function getMockPackageConfig() { - return { - id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', - inputs: [ - { - config: {}, - }, - ], - revision: 1, - version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992) - updated_at: '2020-06-25T16:03:38.159292', - updated_by: 'kibana', - created_at: '2020-06-25T16:03:38.159292', - created_by: 'kibana', - }; -} - -class PackageConfigServiceMock { - public create = jest.fn().mockResolvedValue(getMockPackageConfig()); - public get = jest.fn().mockResolvedValue(getMockPackageConfig()); - public getByIds = jest.fn().mockResolvedValue([getMockPackageConfig()]); - public list = jest.fn().mockResolvedValue({ - items: [getMockPackageConfig()], - total: 1, - page: 1, - perPage: 20, - }); - public update = jest.fn().mockResolvedValue(getMockPackageConfig()); -} - -export function getPackageConfigServiceMock() { - return new PackageConfigServiceMock(); -} - async function mockBuildExceptionListArtifacts( os: string, schemaVersion: string @@ -66,27 +34,23 @@ async function mockBuildExceptionListArtifacts( return [await buildArtifact(exceptions, os, schemaVersion)]; } -// @ts-ignore export class ManifestManagerMock extends ManifestManager { - // @ts-ignore - private buildExceptionListArtifacts = async () => { - return mockBuildExceptionListArtifacts('linux', 'v1'); - }; + protected buildExceptionListArtifacts = jest + .fn() + .mockResolvedValue(mockBuildExceptionListArtifacts('linux', 'v1')); - // @ts-ignore - private getLastDispatchedManifest = jest + public getLastDispatchedManifest = jest .fn() .mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION)); - // @ts-ignore - private getManifestClient = jest + protected getManifestClient = jest .fn() .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); } export const getManifestManagerMock = (opts?: { cache?: ExceptionsCache; - packageConfigService?: PackageConfigServiceMock; + packageConfigService?: jest.Mocked; savedObjectsClient?: ReturnType; }): ManifestManagerMock => { let cache = new ExceptionsCache(5); @@ -94,10 +58,14 @@ export const getManifestManagerMock = (opts?: { cache = opts.cache; } - let packageConfigService = getPackageConfigServiceMock(); + let packageConfigService = createPackageConfigServiceMock(); if (opts?.packageConfigService !== undefined) { packageConfigService = opts.packageConfigService; } + packageConfigService.list = jest.fn().mockResolvedValue({ + total: 1, + items: [createPackageConfigMock()], + }); let savedObjectsClient = savedObjectsClientMock.create(); if (opts?.savedObjectsClient !== undefined) { @@ -107,7 +75,6 @@ export const getManifestManagerMock = (opts?: { const manifestManager = new ManifestManagerMock({ artifactClient: getArtifactClientMock(savedObjectsClient), cache, - // @ts-ignore packageConfigService, exceptionListClient: listMock.getExceptionListClient(), logger: loggingSystemMock.create().get() as jest.Mocked, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index b1cbc41459f15f..d092e7060f8aa9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -6,13 +6,14 @@ import { inflateSync } from 'zlib'; import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks'; import { ArtifactConstants, ManifestConstants, Manifest, ExceptionsCache, } from '../../../lib/artifacts'; -import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock'; +import { getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { @@ -73,15 +74,15 @@ describe('manifest_manager', () => { }); test('ManifestManager can dispatch manifest', async () => { - const packageConfigService = getPackageConfigServiceMock(); + const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); const snapshot = await manifestManager.getSnapshot(); const dispatched = await manifestManager.dispatch(snapshot!.manifest); - expect(dispatched).toEqual(true); + expect(dispatched).toEqual([]); const entries = snapshot!.manifest.getEntries(); const artifact = Object.values(entries)[0].getArtifact(); expect( - packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value + packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value ).toEqual({ manifest_version: ManifestConstants.INITIAL_VERSION, schema_version: 'v1', @@ -115,7 +116,7 @@ describe('manifest_manager', () => { snapshot!.diffs.push(diff); const dispatched = await manifestManager.dispatch(snapshot!.manifest); - expect(dispatched).toEqual(true); + expect(dispatched).toEqual([]); await manifestManager.commit(snapshot!.manifest); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index b9e289cee62af1..c8cad32ab746e2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -61,19 +61,25 @@ export class ManifestManager { /** * Gets a ManifestClient for the provided schemaVersion. * - * @param schemaVersion + * @param schemaVersion The schema version of the manifest. + * @returns {ManifestClient} A ManifestClient scoped to the provided schemaVersion. */ - private getManifestClient(schemaVersion: string) { + protected getManifestClient(schemaVersion: string): ManifestClient { return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); } /** * Builds an array of artifacts (one per supported OS) based on the current - * state of exception-list-agnostic SO's. + * state of exception-list-agnostic SOs. * - * @param schemaVersion + * @param schemaVersion The schema version of the artifact + * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. + * @throws Throws/rejects if there are errors building the list. */ - private async buildExceptionListArtifacts(schemaVersion: string) { + protected async buildExceptionListArtifacts( + schemaVersion: string + ): Promise { + // TODO: should wrap in try/catch? return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce( async (acc: Promise, os) => { const exceptionList = await getFullEndpointExceptionList( @@ -90,13 +96,75 @@ export class ManifestManager { ); } + /** + * Writes new artifact SOs based on provided snapshot. + * + * @param snapshot A ManifestSnapshot to use for writing the artifacts. + * @returns {Promise} Any errors encountered. + */ + private async writeArtifacts(snapshot: ManifestSnapshot): Promise { + const errors: Error[] = []; + for (const diff of snapshot.diffs) { + const artifact = snapshot.manifest.getArtifact(diff.id); + if (artifact === undefined) { + throw new Error( + `Corrupted manifest detected. Diff contained artifact ${diff.id} not in manifest.` + ); + } + + const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64')); + artifact.body = compressedArtifact.toString('base64'); + artifact.encodedSize = compressedArtifact.byteLength; + artifact.compressionAlgorithm = 'zlib'; + artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); + + try { + // Write the artifact SO + await this.artifactClient.createArtifact(artifact); + // Cache the compressed body of the artifact + this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); + } catch (err) { + if (err.status === 409) { + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + // TODO: log error here? + errors.push(err); + } + } + } + return errors; + } + + /** + * Deletes old artifact SOs based on provided snapshot. + * + * @param snapshot A ManifestSnapshot to use for deleting the artifacts. + * @returns {Promise} Any errors encountered. + */ + private async deleteArtifacts(snapshot: ManifestSnapshot): Promise { + const errors: Error[] = []; + for (const diff of snapshot.diffs) { + try { + // Delete the artifact SO + await this.artifactClient.deleteArtifact(diff.id); + // TODO: should we delete the cache entry here? + this.logger.info(`Cleaned up artifact ${diff.id}`); + } catch (err) { + errors.push(err); + } + } + return errors; + } + /** * Returns the last dispatched manifest based on the current state of the * user-artifact-manifest SO. * - * @param schemaVersion + * @param schemaVersion The schema version of the manifest. + * @returns {Promise} The last dispatched manifest, or null if does not exist. + * @throws Throws/rejects if there is an unexpected error retrieving the manifest. */ - private async getLastDispatchedManifest(schemaVersion: string) { + public async getLastDispatchedManifest(schemaVersion: string): Promise { try { const manifestClient = this.getManifestClient(schemaVersion); const manifestSo = await manifestClient.getManifest(); @@ -127,9 +195,11 @@ export class ManifestManager { /** * Snapshots a manifest based on current state of exception-list-agnostic SOs. * - * @param opts TODO + * @param opts Optional parameters for snapshot retrieval. + * @param opts.initialize Initialize a new Manifest when no manifest SO can be retrieved. + * @returns {Promise} A snapshot of the manifest, or null if not initialized. */ - public async getSnapshot(opts?: ManifestSnapshotOpts) { + public async getSnapshot(opts?: ManifestSnapshotOpts): Promise { try { let oldManifest: Manifest | null; @@ -176,71 +246,39 @@ export class ManifestManager { * Creates artifacts that do not yet exist and cleans up old artifacts that have been * superceded by this snapshot. * - * Can be filtered to apply one or both operations. - * - * @param snapshot - * @param diffType + * @param snapshot A ManifestSnapshot to use for sync. + * @returns {Promise} Any errors encountered. */ - public async syncArtifacts(snapshot: ManifestSnapshot, diffType?: 'add' | 'delete') { - const filteredDiffs = snapshot.diffs.reduce((diffs: ManifestDiff[], diff) => { - if (diff.type === diffType || diffType === undefined) { - diffs.push(diff); - } else if (!['add', 'delete'].includes(diff.type)) { - // TODO: replace with io-ts schema - throw new Error(`Unsupported diff type: ${diff.type}`); - } - return diffs; - }, []); - - const adds = filteredDiffs.filter((diff) => { - return diff.type === 'add'; + public async syncArtifacts( + snapshot: ManifestSnapshot, + diffType: 'add' | 'delete' + ): Promise { + const filteredDiffs = snapshot.diffs.filter((diff) => { + return diff.type === diffType; }); - const deletes = filteredDiffs.filter((diff) => { - return diff.type === 'delete'; - }); + const tmpSnapshot = { ...snapshot }; + tmpSnapshot.diffs = filteredDiffs; - for (const diff of adds) { - const artifact = snapshot.manifest.getArtifact(diff.id); - if (artifact === undefined) { - throw new Error( - `Corrupted manifest detected. Diff contained artifact ${diff.id} not in manifest.` - ); - } - const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64')); - artifact.body = compressedArtifact.toString('base64'); - artifact.encodedSize = compressedArtifact.byteLength; - artifact.compressionAlgorithm = 'zlib'; - artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); - - try { - await this.artifactClient.createArtifact(artifact); - } catch (err) { - if (err.status === 409) { - this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); - } else { - throw err; - } - } - // Cache the body of the artifact - this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); + if (diffType === 'add') { + return this.writeArtifacts(tmpSnapshot); + } else if (diffType === 'delete') { + return this.deleteArtifacts(tmpSnapshot); } - for (const diff of deletes) { - await this.artifactClient.deleteArtifact(diff.id); - // TODO: should we delete the cache entry here? - this.logger.info(`Cleaned up artifact ${diff.id}`); - } + return [new Error(`Unsupported diff type: ${diffType}`)]; } /** * Dispatches the manifest by writing it to the endpoint package config. * + * @param manifest The Manifest to dispatch. + * @returns {Promise} Any errors encountered. */ - public async dispatch(manifest: Manifest) { + public async dispatch(manifest: Manifest): Promise { let paging = true; let page = 1; - let success = true; + const errors: Error[] = []; while (paging) { const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { @@ -264,13 +302,10 @@ export class ManifestManager { `Updated package config ${id} with manifest version ${manifest.getVersion()}` ); } catch (err) { - success = false; - this.logger.debug(`Error updating package config ${id}`); - this.logger.error(err); + errors.push(err); } } else { - success = false; - this.logger.debug(`Package config ${id} has no config.`); + errors.push(new Error(`Package config ${id} has no config.`)); } } @@ -278,32 +313,38 @@ export class ManifestManager { page++; } - // TODO: revisit success logic - return success; + return errors; } /** * Commits a manifest to indicate that it has been dispatched. * - * @param manifest + * @param manifest The Manifest to commit. + * @returns {Promise} An error if encountered, or null if successful. */ - public async commit(manifest: Manifest) { - const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); - - // Commit the new manifest - if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) { - await manifestClient.createManifest(manifest.toSavedObject()); - } else { - const version = manifest.getVersion(); - if (version === ManifestConstants.INITIAL_VERSION) { - throw new Error('Updating existing manifest with baseline version. Bad state.'); + public async commit(manifest: Manifest): Promise { + try { + const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); + + // Commit the new manifest + if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) { + await manifestClient.createManifest(manifest.toSavedObject()); + } else { + const version = manifest.getVersion(); + if (version === ManifestConstants.INITIAL_VERSION) { + throw new Error('Updating existing manifest with baseline version. Bad state.'); + } + await manifestClient.updateManifest(manifest.toSavedObject(), { + version, + }); } - await manifestClient.updateManifest(manifest.toSavedObject(), { - version, - }); + + this.logger.info(`Committed manifest ${manifest.getVersion()}`); + } catch (err) { + return err; } - this.logger.info(`Committed manifest ${manifest.getVersion()}`); + return null; } /** diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index db33775abeb0a7..7207bb3fc37b37 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -116,6 +116,48 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { version: policyInfo.packageInfo.version, }, }, + artifact_manifest: { + artifacts: { + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', + decoded_sha256: + 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: + 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + decoded_sha256: + 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: + 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + decoded_sha256: + 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: + 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + }, + manifest_version: 'WzEwNSwxXQ==', + schema_version: 'v1', + }, policy: { linux: { events: { file: false, network: true, process: true },