From b699c706b0c3cdaf78e6cd95379aa6dca8fa75be Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 13 Apr 2020 21:40:05 -0400 Subject: [PATCH 01/52] step 1 to add aggs in the find function of saved object --- .../saved_objects/saved_objects_client.ts | 1 + .../saved_objects/service/lib/aggs_utils.ts | 46 ++++ .../server/saved_objects/service/lib/index.ts | 2 + .../saved_objects/service/lib/repository.ts | 12 ++ .../bucket_aggs/index.ts | 48 +++++ .../saved_object_aggs_types/helpers.test.ts | 196 ++++++++++++++++++ .../lib/saved_object_aggs_types/helpers.ts | 122 +++++++++++ .../lib/saved_object_aggs_types/index.ts | 45 ++++ .../metrics_aggs/index.ts | 74 +++++++ .../service/saved_objects_client.ts | 8 +- src/core/server/saved_objects/types.ts | 3 +- .../common/suites/find.ts | 24 +-- 12 files changed, 567 insertions(+), 14 deletions(-) create mode 100644 src/core/server/saved_objects/service/lib/aggs_utils.ts create mode 100644 src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts create mode 100644 src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts create mode 100644 src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts create mode 100644 src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts create mode 100644 src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index afc77806afb914..78acf1c76d3c4c 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -306,6 +306,7 @@ export class SavedObjectsClient { sortField: 'sort_field', type: 'type', filter: 'filter', + aggs: 'aggs', }; const renamedQuery = renameKeys(renameMap, options); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts new file mode 100644 index 00000000000000..0e7bc130bb3a50 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IndexMapping } from '../../mappings'; + +import { SavedObjectsErrorHelpers } from './errors'; +import { hasFilterKeyError } from './filter_utils'; +import { SavedObjectAggs, validateSavedObjectTypeAggs } from './saved_object_aggs_types'; + +export const validateSavedObjectAggs = ( + allowedTypes: string[], + aggs: SavedObjectAggs, + indexMapping: IndexMapping +) => { + validateSavedObjectTypeAggs(aggs); + validateAggFieldValue(allowedTypes, aggs, indexMapping); +}; + +const validateAggFieldValue = (allowedTypes: string[], aggs: any, indexMapping: IndexMapping) => { + Object.keys(aggs).forEach(key => { + if (key === 'field') { + const error = hasFilterKeyError(key, allowedTypes, indexMapping); + if (error != null) { + throw SavedObjectsErrorHelpers.createBadRequestError(error); + } + } else if (typeof aggs[key] === 'object') { + validateAggFieldValue(allowedTypes, aggs[key], indexMapping); + } + }); +}; diff --git a/src/core/server/saved_objects/service/lib/index.ts b/src/core/server/saved_objects/service/lib/index.ts index e103120388e354..d338e939ffb9c7 100644 --- a/src/core/server/saved_objects/service/lib/index.ts +++ b/src/core/server/saved_objects/service/lib/index.ts @@ -30,3 +30,5 @@ export { } from './scoped_client_provider'; export { SavedObjectsErrorHelpers } from './errors'; + +export { SavedObjectAggs } from './saved_object_aggs_types'; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 5f17c117927638..37a0dcdd3bc222 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -57,6 +57,7 @@ import { } from '../../types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; +import { validateSavedObjectAggs } from './aggs_utils'; // BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository // so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient. @@ -592,6 +593,7 @@ export class SavedObjectsRepository { * @property {Array} [options.fields] * @property {string} [options.namespace] * @property {object} [options.hasReference] - { type, id } + * @property {object} [options.aggs] - see ./saved_object_aggs for more insight * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ async find({ @@ -607,6 +609,7 @@ export class SavedObjectsRepository { namespace, type, filter, + aggs, }: SavedObjectsFindOptions): Promise> { if (!type) { throw SavedObjectsErrorHelpers.createBadRequestError( @@ -647,6 +650,14 @@ export class SavedObjectsRepository { } } + try { + if (aggs) { + validateSavedObjectAggs(allowedTypes, aggs, this._mappings); + } + } catch (e) { + throw e; + } + const esOptions = { index: this.getIndicesForTypes(allowedTypes), size: perPage, @@ -656,6 +667,7 @@ export class SavedObjectsRepository { rest_total_hits_as_int: true, body: { seq_no_primary_term: true, + ...(aggs != null ? { aggs } : {}), ...getSearchDsl(this._mappings, this._registry, { search, defaultSearchOperator, diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts new file mode 100644 index 00000000000000..8be958a94b4c1e --- /dev/null +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as rt from 'io-ts'; + +import { FieldBasicRT } from '../helpers'; + +export const BucketAggsTypeRt = rt.partial({ + filter: rt.type({ + term: rt.record(rt.string, rt.string), + }), + histogram: rt.intersection([ + FieldBasicRT, + rt.type({ interval: rt.number }), + rt.partial({ + min_doc_count: rt.number, + extended_bounds: rt.type({ min: rt.number, max: rt.number }), + keyed: rt.boolean, + missing: rt.number, + order: rt.record(rt.string, rt.literal('asc', 'desc')), + }), + ]), + terms: rt.intersection([ + FieldBasicRT, + rt.partial({ + field: rt.string, + size: rt.number, + show_term_doc_count_error: rt.boolean, + order: rt.record(rt.string, rt.literal('asc', 'desc')), + }), + ]), +}); diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts new file mode 100644 index 00000000000000..1a72f144977838 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts @@ -0,0 +1,196 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as rt from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; + +import { excess } from './helpers'; + +const runDecode = (codec: rt.Type, data: any) => { + const result = codec.decode(data); + return PathReporter.report(result); +}; + +describe('Saved object aggs helpers', () => { + describe('happy path', () => { + test('excess Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.partial({ + max: rt.type({ field: rt.string }), + }) + ) + ); + + expect(runDecode(codec, { aggName: { max: { field: 'hi' } } })).toEqual(['No errors!']); + }); + + test('excess Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.intersection([ + rt.partial({ + max: rt.type({ field: rt.string }), + }), + rt.partial({ + min: rt.type({ field: rt.string }), + }), + ]) + ) + ); + + expect(runDecode(codec, { aggName: { min: { field: 'hi' } } })).toEqual(['No errors!']); + }); + + test('When you intersection as a DictionnaryType', () => { + const codec = excess( + rt.record( + rt.string, + rt.intersection([ + rt.partial({ + max: rt.type({ field: rt.string }), + }), + rt.partial({ + filter: rt.type({ field: rt.string }), + aggs: rt.record( + rt.string, + rt.partial({ + min: rt.type({ field: rt.string }), + }) + ), + }), + ]) + ) + ); + + expect( + runDecode(codec, { + aggName: { filter: { field: 'hi' }, aggs: { aggNewName: { min: { field: 'low' } } } }, + }) + ).toEqual(['No errors!']); + }); + }); + + describe('Errors', () => { + test('throw error when you add an attributes who is not expected for Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.partial({ + max: rt.type({ field: rt.string }), + }) + ) + ); + + expect(runDecode(codec, { aggName: { max: { field: 'hi', script: '' } } })).toEqual([ + 'Invalid value {"aggName":{"max":{"field":"hi","script":""}}} supplied to : { [K in string]: Partial<{ max: { field: string } }> }, excess properties: ["script"]', + ]); + }); + + test('throw error when you add an attributes who is not expected for Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.intersection([ + rt.partial({ + max: rt.type({ field: rt.string }), + }), + rt.partial({ + min: rt.type({ field: rt.string }), + }), + ]) + ) + ); + + expect(runDecode(codec, { aggName: { min: { field: 'hi', script: 'field' } } })).toEqual([ + 'Invalid value {"aggName":{"min":{"field":"hi","script":"field"}}} supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>) }, excess properties: ["script"]', + ]); + }); + + test('throw error when you do not match types for Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.partial({ + max: rt.type({ field: rt.string }), + }) + ) + ); + + expect(runDecode(codec, { aggName: { max: { field: 33 } } })).toEqual([ + 'Invalid value 33 supplied to : { [K in string]: Partial<{ max: { field: string } }> }/aggName: Partial<{ max: { field: string } }>/max: { field: string }/field: string', + ]); + }); + + test('throw error when when you do not match types for Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.intersection([ + rt.partial({ + max: rt.type({ field: rt.string }), + }), + rt.partial({ + min: rt.type({ field: rt.string }), + }), + ]) + ) + ); + + expect(runDecode(codec, { aggName: { min: { field: 33 } } })).toEqual([ + 'Invalid value 33 supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>) }/aggName: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>)/1: Partial<{ min: { field: string } }>/min: { field: string }/field: string', + ]); + }); + + test('throw error when you add an attributes in your second agg who is not expected for Record', () => { + const codec = excess( + rt.record( + rt.string, + rt.intersection([ + rt.partial({ + max: rt.type({ field: rt.string }), + }), + rt.partial({ + filter: rt.type({ field: rt.string }), + aggs: rt.record( + rt.string, + rt.partial({ + min: rt.type({ field: rt.string }), + }) + ), + }), + ]) + ) + ); + + expect( + runDecode(codec, { + aggName: { + filter: { field: 'hi' }, + aggs: { aggNewName: { min: { field: 'low' }, script: 'error' } }, + }, + }) + ).toEqual([ + 'Invalid value {"aggName":{"filter":{"field":"hi"},"aggs":{"aggNewName":{"min":{"field":"low"},"script":"error"}}}} supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ filter: { field: string }, aggs: { [K in string]: Partial<{ min: { field: string } }> } }>) }, excess properties: ["script"]', + ]); + }); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts new file mode 100644 index 00000000000000..0ea5c3d9301bb3 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { either } from 'fp-ts/lib/Either'; +import * as rt from 'io-ts'; +import { failure } from 'io-ts/lib/PathReporter'; +import { get } from 'lodash'; + +type ErrorFactory = (message: string) => Error; + +export const FieldBasicRT = rt.type({ field: rt.string }); + +export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { + throw createError(failure(errors).join('\n')); +}; + +const getProps = (codec: rt.HasProps | rt.RecordC): rt.Props | null => { + if (codec == null) { + return null; + } + switch (codec._tag) { + case 'DictionaryType': + if (codec.codomain.props != null) { + return codec.codomain.props; + } + const types: rt.HasProps[] = codec.codomain.types; + return types.reduce((props, type) => Object.assign(props, getProps(type)), {}); + case 'RefinementType': + case 'ReadonlyType': + return getProps(codec.type); + case 'InterfaceType': + case 'StrictType': + case 'PartialType': + return codec.props; + case 'IntersectionType': + return codec.types.reduce( + (props, type) => Object.assign(props, getProps(type)), + {} + ); + default: + return null; + } +}; + +const getExcessProps = ( + props: rt.Props | rt.RecordC, + r: Record +): string[] => + Object.keys(r).reduce((acc, k) => { + const codecChildren = get>(props, [k]); + const childrenProps = getProps(codecChildren); + const childrenObject = r[k] as Record; + if (codecChildren != null && childrenProps != null && codecChildren._tag === 'DictionaryType') { + const keys = Object.keys(childrenObject); + return [ + ...acc, + ...keys.reduce( + (kAcc, i) => [...kAcc, ...getExcessProps(childrenProps, childrenObject[i])], + [] + ), + ]; + } + if (props.hasOwnProperty(k) && childrenProps != null) { + return [...acc, ...getExcessProps(childrenProps, childrenObject)]; + } else if (!props.hasOwnProperty(k)) { + return [...acc, k]; + } + return acc; + }, []); + +export const excess = (codec: rt.RecordC): rt.InterfaceType => { + const codecProps = getProps(codec); + + const r = new rt.DictionaryType( + codec.name, + codec.is, + (u, c) => + either.chain(codec.validate(u, c), (o: Record) => { + if (codecProps == null) { + return rt.failure( + u, + c, + `Invalid Aggs object ${JSON.stringify(u)} supplied to : ${codec.name}` + ); + } + const keys = Object.keys(o); + const ex = keys.reduce((acc, k) => { + return [...acc, ...getExcessProps(codecProps, o[k])]; + }, []); + + return ex.length > 0 + ? rt.failure( + u, + c, + `Invalid value ${JSON.stringify(u)} supplied to : ${ + codec.name + }, excess properties: ${JSON.stringify(ex)}` + ) + : codec.validate(u, c); + }), + codec.encode, + codec.domain, + codec.codomain + ); + return r as any; +}; diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts new file mode 100644 index 00000000000000..68461ebbe3ee00 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { fold } from 'fp-ts/lib/Either'; +import { identity } from 'fp-ts/lib/function'; +import { pipe } from 'fp-ts/lib/pipeable'; +import * as rt from 'io-ts'; + +import { BucketAggsTypeRt } from './bucket_aggs'; +import { MetricsAggsTypeRt } from './metrics_aggs'; + +import { SavedObjectsErrorHelpers } from '../errors'; +import { excess, throwErrors } from './helpers'; + +const AllAggsRt = rt.intersection([BucketAggsTypeRt, MetricsAggsTypeRt]); + +const SavedObjectAggsRt = rt.record( + rt.string, + rt.intersection([AllAggsRt, rt.partial({ aggs: AllAggsRt })]) +); + +export type SavedObjectAggs = rt.TypeOf; + +export const validateSavedObjectTypeAggs = (aggObjects: SavedObjectAggs) => { + pipe( + excess(SavedObjectAggsRt).decode(aggObjects), + fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) + ); +}; diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts new file mode 100644 index 00000000000000..169d8d5c426a0f --- /dev/null +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as rt from 'io-ts'; + +import { FieldBasicRT } from '../helpers'; + +/* + * Types for Metrics Aggregations + * + * TODO: + * - Extended Stats Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-extendedstats-aggregation.html + * - Geo Bounds Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geobounds-aggregation.html + * - Geo Centroid Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geocentroid-aggregation.html + * - Percentiles Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-percentile-aggregation.html + * - Percentile Ranks Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-percentile-rank-aggregation.html + * - Scripted Metric Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html + * - Stats Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-stats-aggregation.html + * - String Stats Aggregation (x-pack) https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-string-stats-aggregation.html + * - Sum Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-sum-aggregation.html + * - Top Hits Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-top-hits-aggregation.html + * - Value Count Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-valuecount-aggregation.html + * - Median Absolute Deviation Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-median-absolute-deviation-aggregation.html + */ + +const MinMaxAggBodyRt = rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]); + +export const MetricsAggsTypeRt = rt.partial({ + avg: FieldBasicRT, + weighted_avg: rt.intersection([ + rt.type({ + value: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), + weight: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), + }), + rt.partial({ + format: rt.string, + value_type: rt.string, + }), + ]), + cardinality: FieldBasicRT, + max: MinMaxAggBodyRt, + min: MinMaxAggBodyRt, + top_hits: rt.partial({ + explain: rt.boolean, + from: rt.string, + highlight: rt.any, + seq_no_primary_term: rt.boolean, + size: rt.number, + sort: rt.any, + stored_fields: rt.array(rt.string), + version: rt.boolean, + _name: rt.string, + _source: rt.partial({ + includes: rt.array(rt.string), + excludes: rt.array(rt.string), + }), + }), +}); diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 8780f07cc3091b..19fb504509c12a 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ISavedObjectsRepository } from './lib'; +import { ISavedObjectsRepository, SavedObjectAggs } from './lib'; import { SavedObject, SavedObjectReference, @@ -182,6 +182,12 @@ export interface SavedObjectsUpdateResponse references: SavedObjectReference[] | undefined; } +/** + * + * @public + */ +export { SavedObjectAggs }; + /** * * @public diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 9efc82603b1796..44e54119d87f84 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -17,7 +17,7 @@ * under the License. */ -import { SavedObjectsClient } from './service/saved_objects_client'; +import { SavedObjectsClient, SavedObjectAggs } from './service/saved_objects_client'; import { SavedObjectsTypeMappingDefinition, SavedObjectsTypeMappingDefinitions } from './mappings'; import { SavedObjectMigrationMap } from './migrations'; import { PropertyValidators } from './validation'; @@ -82,6 +82,7 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { hasReference?: { type: string; id: string }; defaultSearchOperator?: 'AND' | 'OR'; filter?: string; + aggs?: SavedObjectAggs; } /** diff --git a/x-pack/test/saved_object_api_integration/common/suites/find.ts b/x-pack/test/saved_object_api_integration/common/suites/find.ts index 75d6653365fdf0..2fda8bb0aaab01 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/find.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/find.ts @@ -34,7 +34,7 @@ export interface FindTestCase { failure?: 400 | 403; } -export const getTestCases = (spaceId?: string) => ({ +export const getTestCases = (spaceId?: string): Record => ({ singleNamespaceType: { title: 'find single-namespace type', query: 'type=isolatedtype&fields=title', @@ -46,7 +46,7 @@ export const getTestCases = (spaceId?: string) => ({ ? CASES.SINGLE_NAMESPACE_SPACE_2 : CASES.SINGLE_NAMESPACE_DEFAULT_SPACE, }, - } as FindTestCase, + }, multiNamespaceType: { title: 'find multi-namespace type', query: 'type=sharedtype&fields=title', @@ -58,41 +58,41 @@ export const getTestCases = (spaceId?: string) => ({ ? CASES.MULTI_NAMESPACE_ONLY_SPACE_2 : CASES.MULTI_NAMESPACE_DEFAULT_AND_SPACE_1, }, - } as FindTestCase, + }, namespaceAgnosticType: { title: 'find namespace-agnostic type', query: 'type=globaltype&fields=title', successResult: { savedObjects: CASES.NAMESPACE_AGNOSTIC }, - } as FindTestCase, - hiddenType: { title: 'find hidden type', query: 'type=hiddentype&fields=name' } as FindTestCase, - unknownType: { title: 'find unknown type', query: 'type=wigwags' } as FindTestCase, + }, + hiddenType: { title: 'find hidden type', query: 'type=hiddentype&fields=name' }, + unknownType: { title: 'find unknown type', query: 'type=wigwags' }, pageBeyondTotal: { title: 'find page beyond total', query: 'type=isolatedtype&page=100&per_page=100', successResult: { page: 100, perPage: 100, total: 1, savedObjects: [] }, - } as FindTestCase, + }, unknownSearchField: { title: 'find unknown search field', query: 'type=url&search_fields=a', - } as FindTestCase, + }, filterWithNamespaceAgnosticType: { title: 'filter with namespace-agnostic type', query: 'type=globaltype&filter=globaltype.attributes.title:*global*', successResult: { savedObjects: CASES.NAMESPACE_AGNOSTIC }, - } as FindTestCase, + }, filterWithHiddenType: { title: 'filter with hidden type', query: `type=hiddentype&fields=name&filter=hiddentype.attributes.title:'hello'`, - } as FindTestCase, + }, filterWithUnknownType: { title: 'filter with unknown type', query: `type=wigwags&filter=wigwags.attributes.title:'unknown'`, - } as FindTestCase, + }, filterWithDisallowedType: { title: 'filter with disallowed type', query: `type=globaltype&filter=dashboard.title:'Requests'`, failure: 400, - } as FindTestCase, + }, }); export const createRequest = ({ query }: FindTestCase) => ({ query }); const getTestTitle = ({ failure, title }: FindTestCase) => { From 3819cd520059263e30287e98b360bb278f8a8320 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Wed, 15 Apr 2020 15:53:24 -0400 Subject: [PATCH 02/52] setp 2 - add specific unit test to aggs + fix bug found during integrations --- .../saved_objects/saved_objects_client.ts | 9 ++- src/core/server/saved_objects/routes/find.ts | 2 + .../saved_objects/service/lib/aggs_utils.ts | 28 +++++--- .../saved_objects/service/lib/repository.ts | 13 ++-- .../service/saved_objects_client.ts | 7 +- src/core/server/saved_objects/types.ts | 2 +- .../apis/saved_objects/find.js | 70 +++++++++++++++++++ 7 files changed, 112 insertions(+), 19 deletions(-) diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 78acf1c76d3c4c..90bdc7ecb37514 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -106,7 +106,9 @@ export interface SavedObjectsBatchResponse { * * @public */ -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { +export interface SavedObjectsFindResponsePublic + extends SavedObjectsBatchResponse { + aggregations?: A; total: number; perPage: number; page: number; @@ -291,7 +293,7 @@ export class SavedObjectsClient { * @property {object} [options.hasReference] - { type, id } * @returns A find result with objects matching the specified search. */ - public find = ( + public find = ( options: SavedObjectsFindOptions ): Promise> => { const path = this.getPath(['_find']); @@ -323,13 +325,14 @@ export class SavedObjectsClient { SavedObjectsFindResponsePublic >( { + aggregations: 'aggregations', saved_objects: 'savedObjects', total: 'total', per_page: 'perPage', page: 'page', }, resp - ) as SavedObjectsFindResponsePublic; + ) as SavedObjectsFindResponsePublic; }); }; diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 5c1c2c9a9ab871..4af5acd585097b 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -45,6 +45,7 @@ export const registerFindRoute = (router: IRouter) => { ), fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), filter: schema.maybe(schema.string()), + aggs: schema.maybe(schema.string()), }), }, }, @@ -62,6 +63,7 @@ export const registerFindRoute = (router: IRouter) => { hasReference: query.has_reference, fields: typeof query.fields === 'string' ? [query.fields] : query.fields, filter: query.filter, + aggs: query.aggs, }); return res.ok({ body: result }); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index 0e7bc130bb3a50..21084c1ec2c84b 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -23,24 +23,36 @@ import { SavedObjectsErrorHelpers } from './errors'; import { hasFilterKeyError } from './filter_utils'; import { SavedObjectAggs, validateSavedObjectTypeAggs } from './saved_object_aggs_types'; -export const validateSavedObjectAggs = ( +export const validateGetSavedObjectAggs = ( allowedTypes: string[], aggs: SavedObjectAggs, indexMapping: IndexMapping -) => { +): SavedObjectAggs => { validateSavedObjectTypeAggs(aggs); - validateAggFieldValue(allowedTypes, aggs, indexMapping); + return validateGetAggFieldValue(allowedTypes, aggs, indexMapping); }; -const validateAggFieldValue = (allowedTypes: string[], aggs: any, indexMapping: IndexMapping) => { - Object.keys(aggs).forEach(key => { +const validateGetAggFieldValue = ( + allowedTypes: string[], + aggs: any, + indexMapping: IndexMapping +) => { + return Object.keys(aggs).reduce((acc, key) => { if (key === 'field') { - const error = hasFilterKeyError(key, allowedTypes, indexMapping); + const error = hasFilterKeyError(aggs[key], allowedTypes, indexMapping); if (error != null) { throw SavedObjectsErrorHelpers.createBadRequestError(error); } + return { + ...acc, + [key]: aggs[key].replace('.attributes', ''), + }; } else if (typeof aggs[key] === 'object') { - validateAggFieldValue(allowedTypes, aggs[key], indexMapping); + return { ...acc, [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping) }; } - }); + return { + ...acc, + [key]: aggs[key], + }; + }, {}); }; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 37a0dcdd3bc222..78a5e43ee543d2 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -47,6 +47,7 @@ import { SavedObjectsDeleteOptions, SavedObjectsAddToNamespacesOptions, SavedObjectsDeleteFromNamespacesOptions, + SavedObjectAggs, } from '../saved_objects_client'; import { SavedObject, @@ -57,7 +58,7 @@ import { } from '../../types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; -import { validateSavedObjectAggs } from './aggs_utils'; +import { validateGetSavedObjectAggs } from './aggs_utils'; // BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository // so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient. @@ -596,7 +597,7 @@ export class SavedObjectsRepository { * @property {object} [options.aggs] - see ./saved_object_aggs for more insight * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ - async find({ + async find({ search, defaultSearchOperator = 'OR', searchFields, @@ -610,7 +611,7 @@ export class SavedObjectsRepository { type, filter, aggs, - }: SavedObjectsFindOptions): Promise> { + }: SavedObjectsFindOptions): Promise> { if (!type) { throw SavedObjectsErrorHelpers.createBadRequestError( 'options.type must be a string or an array of strings' @@ -650,9 +651,10 @@ export class SavedObjectsRepository { } } + let aggsObject = null; try { if (aggs) { - validateSavedObjectAggs(allowedTypes, aggs, this._mappings); + aggsObject = validateGetSavedObjectAggs(allowedTypes, JSON.parse(aggs), this._mappings); } } catch (e) { throw e; @@ -667,7 +669,7 @@ export class SavedObjectsRepository { rest_total_hits_as_int: true, body: { seq_no_primary_term: true, - ...(aggs != null ? { aggs } : {}), + ...(aggsObject != null ? { aggs: aggsObject } : {}), ...getSearchDsl(this._mappings, this._registry, { search, defaultSearchOperator, @@ -696,6 +698,7 @@ export class SavedObjectsRepository { } return { + ...(response.aggregations != null ? { aggregations: response.aggregations } : {}), page, per_page: perPage, total: response.hits.total, diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 19fb504509c12a..047048c68179e0 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -87,7 +87,8 @@ export interface SavedObjectsBulkResponse { * * @public */ -export interface SavedObjectsFindResponse { +export interface SavedObjectsFindResponse { + aggregations?: A; saved_objects: Array>; total: number; per_page: number; @@ -243,7 +244,9 @@ export class SavedObjectsClient { * * @param options */ - async find(options: SavedObjectsFindOptions): Promise> { + async find( + options: SavedObjectsFindOptions + ): Promise> { return await this._repository.find(options); } diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 44e54119d87f84..414487012cc305 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -82,7 +82,7 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { hasReference?: { type: string; id: string }; defaultSearchOperator?: 'AND' | 'OR'; filter?: string; - aggs?: SavedObjectAggs; + aggs?: string; } /** diff --git a/test/api_integration/apis/saved_objects/find.js b/test/api_integration/apis/saved_objects/find.js index 54a19602fd414a..7db7e5789816e1 100644 --- a/test/api_integration/apis/saved_objects/find.js +++ b/test/api_integration/apis/saved_objects/find.js @@ -182,6 +182,76 @@ export default function({ getService }) { }); })); }); + + describe('with a aggs', () => { + it('should return 200 with a valid response', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'visualization.attributes.version' } }, + }) + )}` + ) + .expect(200) + .then(resp => { + expect(resp.body).to.eql({ + aggregations: { + type_count: { + value: 1, + }, + }, + page: 1, + per_page: 0, + saved_objects: [], + total: 1, + }); + })); + + it('wrong type should return 400 with Bad Request', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'dashboard.attributes.version' } }, + }) + )}` + ) + .expect(400) + .then(resp => { + console.log('body', JSON.stringify(resp.body)); + expect(resp.body).to.eql({ + error: 'Bad Request', + message: 'This type dashboard is not allowed: Bad Request', + statusCode: 400, + }); + })); + + it('adding a wrong attributes should return 400 with Bad Request', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { + max: { + field: 'dashboard.attributes.version', + script: 'Oh yes I am going to a script', + }, + }, + }) + )}` + ) + .expect(400) + .then(resp => { + console.log('body', JSON.stringify(resp.body)); + expect(resp.body).to.eql({ + error: 'Bad Request', + message: + 'Invalid value {"type_count":{"max":{"field":"dashboard.attributes.version","script":"Oh yes I am going to a script"}}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: ["script"]: Bad Request', + statusCode: 400, + }); + })); + }); }); describe('without kibana index', () => { From 1c4bac9e09b1020ab64d5b514653c15516c4d933 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 20 Apr 2020 12:26:40 -0400 Subject: [PATCH 03/52] step 3 - add security api_integration arounds aggs --- .../saved_objects/spaces/data.json | 3 +- .../saved_objects/spaces/mappings.json | 3 ++ .../mappings.json | 3 ++ .../common/suites/find.ts | 45 ++++++++++++++++++- .../security_and_spaces/apis/find.ts | 4 ++ .../security_only/apis/find.ts | 4 ++ 6 files changed, 60 insertions(+), 2 deletions(-) diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json index d2c14189e2529a..80a27f1915997f 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json @@ -338,7 +338,8 @@ "index": ".kibana", "source": { "globaltype": { - "title": "My favorite global object" + "title": "My favorite global object", + "version": 1 }, "type": "globaltype", "updated_at": "2017-09-21T18:59:16.270Z" diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index 7b5b1d86f6bcc8..2c1023b2854b65 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -90,6 +90,9 @@ } }, "type": "text" + }, + "version": { + "type": "integer" } } }, diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json index 64d309b4209a20..f2b4b58369e1a9 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json @@ -9,6 +9,9 @@ "ignore_above": 2048 } } + }, + "version": { + "type": "integer" } } } diff --git a/x-pack/test/saved_object_api_integration/common/suites/find.ts b/x-pack/test/saved_object_api_integration/common/suites/find.ts index 2fda8bb0aaab01..55e39a5f89a17f 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/find.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/find.ts @@ -32,6 +32,7 @@ export interface FindTestCase { total?: number; }; failure?: 400 | 403; + typeUseInQueryField?: string; } export const getTestCases = (spaceId?: string): Record => ({ @@ -79,19 +80,61 @@ export const getTestCases = (spaceId?: string): Record => title: 'filter with namespace-agnostic type', query: 'type=globaltype&filter=globaltype.attributes.title:*global*', successResult: { savedObjects: CASES.NAMESPACE_AGNOSTIC }, + typeUseInQueryField: 'globaltype', }, filterWithHiddenType: { title: 'filter with hidden type', query: `type=hiddentype&fields=name&filter=hiddentype.attributes.title:'hello'`, + typeUseInQueryField: 'hiddentype', }, filterWithUnknownType: { title: 'filter with unknown type', query: `type=wigwags&filter=wigwags.attributes.title:'unknown'`, + typeUseInQueryField: 'wigwags', }, filterWithDisallowedType: { title: 'filter with disallowed type', query: `type=globaltype&filter=dashboard.title:'Requests'`, failure: 400, + typeUseInQueryField: 'dashboard', + }, + aggsWithNamespaceAgnosticType: { + title: 'aggs with namespace-agnostic type', + query: `type=globaltype&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'globaltype.attributes.version' } }, + }) + )}`, + successResult: { savedObjects: CASES.NAMESPACE_AGNOSTIC }, + typeUseInQueryField: 'globaltype', + }, + aggsWithHiddenType: { + title: 'aggs with hidden type', + query: `type=hiddentype&fields=name&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'hiddentype.attributes.title' } }, + }) + )}`, + typeUseInQueryField: 'hiddentype', + }, + aggsWithUnknownType: { + title: 'aggs with unknown type', + query: `type=wigwags&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'wigwags.attributes.version' } }, + }) + )}`, + typeUseInQueryField: 'wigwags', + }, + aggsWithDisallowedType: { + title: 'aggs with disallowed type', + query: `type=globaltype&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'dashboard.attributes.version' } }, + }) + )}`, + failure: 400, + typeUseInQueryField: 'dashboard', }, }); export const createRequest = ({ query }: FindTestCase) => ({ query }); @@ -116,7 +159,7 @@ export function findTestSuiteFactory(esArchiver: any, supertest: SuperTest) const type = parsedQuery.type; await expectForbidden(type)(response); } else if (failure === 400) { - const type = (parsedQuery.filter as string).split('.')[0]; + const type = testCase.typeUseInQueryField ?? 'unknown type'; expect(response.body.error).to.eql('Bad Request'); expect(response.body.statusCode).to.eql(failure); expect(response.body.message).to.eql(`This type ${type} is not allowed: Bad Request`); diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts index 7c16c01d203c05..f415c2e5e8650f 100644 --- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts @@ -19,12 +19,16 @@ const createTestCases = (spaceId: string) => { cases.unknownSearchField, cases.filterWithNamespaceAgnosticType, cases.filterWithDisallowedType, + cases.aggsWithNamespaceAgnosticType, + cases.aggsWithDisallowedType, ]; const hiddenAndUnknownTypes = [ cases.hiddenType, cases.unknownType, cases.filterWithHiddenType, cases.filterWithUnknownType, + cases.aggsWithHiddenType, + cases.aggsWithUnknownType, ]; const allTypes = normalTypes.concat(hiddenAndUnknownTypes); return { normalTypes, hiddenAndUnknownTypes, allTypes }; diff --git a/x-pack/test/saved_object_api_integration/security_only/apis/find.ts b/x-pack/test/saved_object_api_integration/security_only/apis/find.ts index 97513783b94b94..3fef638fcb08ee 100644 --- a/x-pack/test/saved_object_api_integration/security_only/apis/find.ts +++ b/x-pack/test/saved_object_api_integration/security_only/apis/find.ts @@ -19,12 +19,16 @@ const createTestCases = () => { cases.unknownSearchField, cases.filterWithNamespaceAgnosticType, cases.filterWithDisallowedType, + cases.aggsWithNamespaceAgnosticType, + cases.aggsWithDisallowedType, ]; const hiddenAndUnknownTypes = [ cases.hiddenType, cases.unknownType, cases.filterWithHiddenType, cases.filterWithUnknownType, + cases.aggsWithHiddenType, + cases.aggsWithUnknownType, ]; const allTypes = normalTypes.concat(hiddenAndUnknownTypes); return { normalTypes, hiddenAndUnknownTypes, allTypes }; From 6feca9b3a72b54405024e09a5c65b5313f7d43ba Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 20 Apr 2020 12:58:44 -0400 Subject: [PATCH 04/52] fix types --- src/core/server/saved_objects/service/lib/aggs_utils.ts | 2 +- src/core/server/saved_objects/service/lib/repository.ts | 1 - src/core/server/saved_objects/types.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index 21084c1ec2c84b..b3850498ee9584 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -36,7 +36,7 @@ const validateGetAggFieldValue = ( allowedTypes: string[], aggs: any, indexMapping: IndexMapping -) => { +): SavedObjectAggs => { return Object.keys(aggs).reduce((acc, key) => { if (key === 'field') { const error = hasFilterKeyError(aggs[key], allowedTypes, indexMapping); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 78a5e43ee543d2..9f7d465749c3f2 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -47,7 +47,6 @@ import { SavedObjectsDeleteOptions, SavedObjectsAddToNamespacesOptions, SavedObjectsDeleteFromNamespacesOptions, - SavedObjectAggs, } from '../saved_objects_client'; import { SavedObject, diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 414487012cc305..6e762b6180dce1 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -17,7 +17,7 @@ * under the License. */ -import { SavedObjectsClient, SavedObjectAggs } from './service/saved_objects_client'; +import { SavedObjectsClient } from './service/saved_objects_client'; import { SavedObjectsTypeMappingDefinition, SavedObjectsTypeMappingDefinitions } from './mappings'; import { SavedObjectMigrationMap } from './migrations'; import { PropertyValidators } from './validation'; From 8dda82105188348607c4df22235fa1f87b3b77bc Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 20 Apr 2020 16:09:57 -0400 Subject: [PATCH 05/52] unit test added for aggs_utils --- .../service/lib/aggs_utils.test.ts | 103 ++++++++++++++++++ .../service/lib/filter_utils.test.ts | 2 +- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 src/core/server/saved_objects/service/lib/aggs_utils.test.ts diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts new file mode 100644 index 00000000000000..1bbe3443bdc228 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { validateGetSavedObjectAggs } from './aggs_utils'; +import { mockMappings } from './filter_utils.test'; + +describe('Filter Utils', () => { + describe('#validateGetSavedObjectAggs', () => { + test('Validate a simple aggregations', () => { + expect( + validateGetSavedObjectAggs( + ['foo'], + { aggName: { max: { field: 'foo.attributes.bytes' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + max: { + field: 'foo.bytes', + }, + }, + }); + }); + test('Validate a nested simple aggregations', () => { + expect( + validateGetSavedObjectAggs( + ['alert'], + { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', + }, + }, + }); + }); + + test('Throw an error when types is not allowed', () => { + expect(() => { + validateGetSavedObjectAggs( + ['alert'], + { + aggName: { + max: { field: 'foo.attributes.bytes' }, + }, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed: Bad Request"`); + }); + + test('Throw an error when aggregation is not defined in SavedObjectAggs', () => { + expect(() => { + validateGetSavedObjectAggs( + ['foo'], + { + aggName: { + MySuperAgg: { field: 'foo.attributes.bytes' }, + }, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid value {\\"aggName\\":{\\"MySuperAgg\\":{\\"field\\":\\"foo.attributes.bytes\\"}}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: [\\"MySuperAgg\\"]: Bad Request"` + ); + }); + + test('Throw an error when you add attributes who are not defined in SavedObjectAggs', () => { + expect(() => { + validateGetSavedObjectAggs( + ['alert'], + { + aggName: { + cardinality: { field: 'alert.attributes.actions.group' }, + script: 'I want to access that I should not', + }, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid value {\\"aggName\\":{\\"cardinality\\":{\\"field\\":\\"alert.attributes.actions.group\\"},\\"script\\":\\"I want to access that I should not\\"}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: [\\"script\\"]: Bad Request"` + ); + }); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index 4d9bcdda3c8ae9..df7a40081fabfd 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -21,7 +21,7 @@ import { esKuery } from '../../../../../plugins/data/server'; import { validateFilterKueryNode, validateConvertFilterToKueryNode } from './filter_utils'; -const mockMappings = { +export const mockMappings = { properties: { updatedAt: { type: 'date', From 44dcb25e24e29458b0ef2f9ad9c22f50f97c0b6c Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Tue, 21 Apr 2020 21:23:00 -0400 Subject: [PATCH 06/52] add documentation --- docs/api/saved-objects/find.asciidoc | 4 ++++ ...bana-plugin-core-server.savedobjectsclient.find.md | 4 ++-- ...plugin-core-server.savedobjectsfindoptions.aggs.md | 11 +++++++++++ ...bana-plugin-core-server.savedobjectsfindoptions.md | 1 + ...re-server.savedobjectsfindresponse.aggregations.md | 11 +++++++++++ ...ana-plugin-core-server.savedobjectsfindresponse.md | 3 ++- ...-plugin-core-server.savedobjectsrepository.find.md | 6 +++--- ...ibana-plugin-core-server.savedobjectsrepository.md | 2 +- src/core/server/server.api.md | 10 +++++++--- 9 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md diff --git a/docs/api/saved-objects/find.asciidoc b/docs/api/saved-objects/find.asciidoc index 93e60be5d49239..5ea1762983f904 100644 --- a/docs/api/saved-objects/find.asciidoc +++ b/docs/api/saved-objects/find.asciidoc @@ -54,6 +54,10 @@ experimental[] Retrieve a paginated set of {kib} saved objects by various condit It should look like that savedObjectType.attributes.title: "myTitle". However, If you used a direct attribute of a saved object like `updatedAt`, you will have to define your filter like that savedObjectType.updatedAt > 2018-12-22. +`aggs`:: + (Optional, string) The aggs will support aggregation string with the caveat that your field from the aggregation will have the attribute from your type saved object, + it should look like this: savedObjectType.attributes.field. However, If you use a direct attribute of a saved object like updatedAt, you will have to define your filter like this: savedObjectType.updatedAt. + NOTE: As objects change in {kib}, the results on each page of the response also change. Use the find API for traditional paginated results, but avoid using it to export large amounts of data. diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md index 9a4c3df5d2d920..56d76125108d1d 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md @@ -9,7 +9,7 @@ Find all SavedObjects matching the search query Signature: ```typescript -find(options: SavedObjectsFindOptions): Promise>; +find(options: SavedObjectsFindOptions): Promise>; ``` ## Parameters @@ -20,5 +20,5 @@ find(options: SavedObjectsFindOptions): PromiseReturns: -`Promise>` +`Promise>` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md new file mode 100644 index 00000000000000..46b4936295c508 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) > [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) + +## SavedObjectsFindOptions.aggs property + +Signature: + +```typescript +aggs?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index 7421f4282ec933..0481c581c208c8 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -15,6 +15,7 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions | Property | Type | Description | | --- | --- | --- | +| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | string | | | [defaultSearchOperator](./kibana-plugin-core-server.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | | | [fields](./kibana-plugin-core-server.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md new file mode 100644 index 00000000000000..17a899f4c8280e --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResponse](./kibana-plugin-core-server.savedobjectsfindresponse.md) > [aggregations](./kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md) + +## SavedObjectsFindResponse.aggregations property + +Signature: + +```typescript +aggregations?: A; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md index a1b1a7a056206d..e1f776f83193ff 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md @@ -11,13 +11,14 @@ Return type of the Saved Objects `find()` method. Signature: ```typescript -export interface SavedObjectsFindResponse +export interface SavedObjectsFindResponse ``` ## Properties | Property | Type | Description | | --- | --- | --- | +| [aggregations](./kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md) | A | | | [page](./kibana-plugin-core-server.savedobjectsfindresponse.page.md) | number | | | [per\_page](./kibana-plugin-core-server.savedobjectsfindresponse.per_page.md) | number | | | [saved\_objects](./kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md) | Array<SavedObject<T>> | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md index 22222061b30774..81e1bb5f1a0334 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md @@ -7,18 +7,18 @@ Signature: ```typescript -find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, }: SavedObjectsFindOptions): Promise>; +find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, aggs, }: SavedObjectsFindOptions): Promise>; ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| { search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, } | SavedObjectsFindOptions | | +| { search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, aggs, } | SavedObjectsFindOptions | | Returns: -`Promise>` +`Promise>` {promise} - { saved\_objects: \[{ id, type, version, attributes }\], total, per\_page, page } diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md index bd86ff3abbe9b5..e77537fc999224 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md @@ -23,7 +23,7 @@ export declare class SavedObjectsRepository | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.delete.md) | | Deletes an object | | [deleteByNamespace(namespace, options)](./kibana-plugin-core-server.savedobjectsrepository.deletebynamespace.md) | | Deletes all objects from the provided namespace. | | [deleteFromNamespaces(type, id, namespaces, options)](./kibana-plugin-core-server.savedobjectsrepository.deletefromnamespaces.md) | | Removes one or more namespaces from a given multi-namespace saved object. If no namespaces remain, the saved object is deleted entirely. This method and \[addToNamespaces\][SavedObjectsRepository.addToNamespaces()](./kibana-plugin-core-server.savedobjectsrepository.addtonamespaces.md) are the only ways to change which Spaces a multi-namespace saved object is shared to. | -| [find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, })](./kibana-plugin-core-server.savedobjectsrepository.find.md) | | | +| [find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, aggs, })](./kibana-plugin-core-server.savedobjectsrepository.find.md) | | | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.get.md) | | Gets a single object | | [incrementCounter(type, id, counterFieldName, options)](./kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md) | | Increases a counter field by one. Creates the document if one doesn't exist for the given id. | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsrepository.update.md) | | Updates an object | diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 7ca5c75f19e8ff..b4ca1fb65c72d0 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1784,7 +1784,7 @@ export class SavedObjectsClient { static errors: typeof SavedObjectsErrorHelpers; // (undocumented) errors: typeof SavedObjectsErrorHelpers; - find(options: SavedObjectsFindOptions): Promise>; + find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; update(type: string, id: string, attributes: Partial, options?: SavedObjectsUpdateOptions): Promise>; } @@ -1956,6 +1956,8 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { + // (undocumented) + aggs?: string; // (undocumented) defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; @@ -1981,7 +1983,9 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { } // @public -export interface SavedObjectsFindResponse { +export interface SavedObjectsFindResponse { + // (undocumented) + aggregations?: A; // (undocumented) page: number; // (undocumented) @@ -2170,7 +2174,7 @@ export class SavedObjectsRepository { deleteByNamespace(namespace: string, options?: SavedObjectsDeleteByNamespaceOptions): Promise; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise<{}>; // (undocumented) - find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, }: SavedObjectsFindOptions): Promise>; + find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, aggs, }: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFieldName: string, options?: SavedObjectsIncrementCounterOptions): Promise<{ id: string; From 0f57677b8a9b8cb9a05a089d6416c1e0194b25d5 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 27 Apr 2020 12:29:27 -0400 Subject: [PATCH 07/52] fix docs --- .../core/server/kibana-plugin-core-server.md | 1 + ...gin-core-server.savedobjectunsanitizeddoc.md | 13 +++++++++++++ x-pack/plugins/siem/index.scss | 17 +++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectunsanitizeddoc.md create mode 100644 x-pack/plugins/siem/index.scss diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index b8cffb36b821a5..a775c76fece639 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -270,6 +270,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. | | [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) | Describe a [saved object type mapping](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md) field.Please refer to [elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) For the mapping documentation | | [SavedObjectsNamespaceType](./kibana-plugin-core-server.savedobjectsnamespacetype.md) | The namespace type dictates how a saved object can be interacted in relation to namespaces. Each type is mutually exclusive: \* single (default): this type of saved object is namespace-isolated, e.g., it exists in only one namespace. \* multiple: this type of saved object is shareable, e.g., it can exist in one or more namespaces. \* agnostic: this type of saved object is global.Note: do not write logic that uses this value directly; instead, use the appropriate accessors in the [type registry](./kibana-plugin-core-server.savedobjecttyperegistry.md). | +| [SavedObjectUnsanitizedDoc](./kibana-plugin-core-server.savedobjectunsanitizeddoc.md) | We want to have two types, one that guarantees a "references" attribute will exist and one that allows it to be null. Since we're not migrating all the saved objects to have a "references" array, we need to support the scenarios where it may be missing (ex migrations). | | [ScopeableRequest](./kibana-plugin-core-server.scopeablerequest.md) | A user credentials container. It accommodates the necessary auth credentials to impersonate the current user.See [KibanaRequest](./kibana-plugin-core-server.kibanarequest.md). | | [ServiceStatusLevel](./kibana-plugin-core-server.servicestatuslevel.md) | A convenience type that represents the union of each value in [ServiceStatusLevels](./kibana-plugin-core-server.servicestatuslevels.md). | | [SharedGlobalConfig](./kibana-plugin-core-server.sharedglobalconfig.md) | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectunsanitizeddoc.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectunsanitizeddoc.md new file mode 100644 index 00000000000000..fe4eaecd003d09 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectunsanitizeddoc.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectUnsanitizedDoc](./kibana-plugin-core-server.savedobjectunsanitizeddoc.md) + +## SavedObjectUnsanitizedDoc type + +We want to have two types, one that guarantees a "references" attribute will exist and one that allows it to be null. Since we're not migrating all the saved objects to have a "references" array, we need to support the scenarios where it may be missing (ex migrations). + +Signature: + +```typescript +export declare type SavedObjectUnsanitizedDoc = SavedObjectDoc & Partial; +``` diff --git a/x-pack/plugins/siem/index.scss b/x-pack/plugins/siem/index.scss new file mode 100644 index 00000000000000..8b2f6d3cb61563 --- /dev/null +++ b/x-pack/plugins/siem/index.scss @@ -0,0 +1,17 @@ +/* GIS plugin styles */ + +// Import the EUI global scope so we can use EUI constants +@import 'src/legacy/ui/public/styles/_styling_constants'; + +// Prefix all styles with "map" to avoid conflicts. +// Examples +// mapChart +// mapChart__legend +// mapChart__legend--small +// mapChart__legend-isLoading + +@import 'main'; +@import 'mapbox_hacks'; +@import 'connected_components/index'; +@import 'components/index'; +@import 'layers/index'; From 817a3ef7a80764399c827555dd09dfa9bab787ef Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 6 Jul 2020 21:25:48 -0400 Subject: [PATCH 08/52] review I --- src/core/server/saved_objects/routes/find.ts | 2 +- .../saved_objects/service/lib/aggs_utils.test.ts | 9 +++++---- .../server/saved_objects/service/lib/repository.ts | 2 +- .../lib/saved_object_aggs_types/helpers.test.ts | 6 +++--- .../service/lib/saved_object_aggs_types/helpers.ts | 12 +++--------- .../service/lib/saved_object_aggs_types/index.ts | 1 + src/core/server/saved_objects/types.ts | 4 ++-- 7 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 4af5acd585097b..ae728e8e449fa2 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -63,7 +63,7 @@ export const registerFindRoute = (router: IRouter) => { hasReference: query.has_reference, fields: typeof query.fields === 'string' ? [query.fields] : query.fields, filter: query.filter, - aggs: query.aggs, + aggs: query.aggs != null ? JSON.parse(query.aggs) : undefined, }); return res.ok({ body: result }); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index 1bbe3443bdc228..fcc32d92b63634 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -19,6 +19,7 @@ import { validateGetSavedObjectAggs } from './aggs_utils'; import { mockMappings } from './filter_utils.test'; +import { SavedObjectAggs } from './saved_object_aggs_types'; describe('Filter Utils', () => { describe('#validateGetSavedObjectAggs', () => { @@ -75,11 +76,11 @@ describe('Filter Utils', () => { aggName: { MySuperAgg: { field: 'foo.attributes.bytes' }, }, - }, + } as SavedObjectAggs, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid value {\\"aggName\\":{\\"MySuperAgg\\":{\\"field\\":\\"foo.attributes.bytes\\"}}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: [\\"MySuperAgg\\"]: Bad Request"` + `"Invalid value {\\"aggName\\":{\\"MySuperAgg\\":{\\"field\\":\\"foo.attributes.bytes\\"}}}, excess properties: [\\"MySuperAgg\\"]: Bad Request"` ); }); @@ -92,11 +93,11 @@ describe('Filter Utils', () => { cardinality: { field: 'alert.attributes.actions.group' }, script: 'I want to access that I should not', }, - }, + } as SavedObjectAggs, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid value {\\"aggName\\":{\\"cardinality\\":{\\"field\\":\\"alert.attributes.actions.group\\"},\\"script\\":\\"I want to access that I should not\\"}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: [\\"script\\"]: Bad Request"` + `"Invalid value {\\"aggName\\":{\\"cardinality\\":{\\"field\\":\\"alert.attributes.actions.group\\"},\\"script\\":\\"I want to access that I should not\\"}}, excess properties: [\\"script\\"]: Bad Request"` ); }); }); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index f95a4024d4d786..48464041377c92 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -640,7 +640,7 @@ export class SavedObjectsRepository { let aggsObject = null; try { if (aggs) { - aggsObject = validateGetSavedObjectAggs(allowedTypes, JSON.parse(aggs), this._mappings); + aggsObject = validateGetSavedObjectAggs(allowedTypes, aggs, this._mappings); } } catch (e) { throw e; diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts index 1a72f144977838..2feb7fa6c7f7e3 100644 --- a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts @@ -101,7 +101,7 @@ describe('Saved object aggs helpers', () => { ); expect(runDecode(codec, { aggName: { max: { field: 'hi', script: '' } } })).toEqual([ - 'Invalid value {"aggName":{"max":{"field":"hi","script":""}}} supplied to : { [K in string]: Partial<{ max: { field: string } }> }, excess properties: ["script"]', + 'Invalid value {"aggName":{"max":{"field":"hi","script":""}}}, excess properties: ["script"]', ]); }); @@ -121,7 +121,7 @@ describe('Saved object aggs helpers', () => { ); expect(runDecode(codec, { aggName: { min: { field: 'hi', script: 'field' } } })).toEqual([ - 'Invalid value {"aggName":{"min":{"field":"hi","script":"field"}}} supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>) }, excess properties: ["script"]', + 'Invalid value {"aggName":{"min":{"field":"hi","script":"field"}}}, excess properties: ["script"]', ]); }); @@ -189,7 +189,7 @@ describe('Saved object aggs helpers', () => { }, }) ).toEqual([ - 'Invalid value {"aggName":{"filter":{"field":"hi"},"aggs":{"aggNewName":{"min":{"field":"low"},"script":"error"}}}} supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ filter: { field: string }, aggs: { [K in string]: Partial<{ min: { field: string } }> } }>) }, excess properties: ["script"]', + 'Invalid value {"aggName":{"filter":{"field":"hi"},"aggs":{"aggNewName":{"min":{"field":"low"},"script":"error"}}}}, excess properties: ["script"]', ]); }); }); diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts index 0ea5c3d9301bb3..6475d97ec6e797 100644 --- a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts +++ b/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts @@ -63,7 +63,7 @@ const getExcessProps = ( r: Record ): string[] => Object.keys(r).reduce((acc, k) => { - const codecChildren = get>(props, [k]); + const codecChildren = get(props, [k]); const childrenProps = getProps(codecChildren); const childrenObject = r[k] as Record; if (codecChildren != null && childrenProps != null && codecChildren._tag === 'DictionaryType') { @@ -93,11 +93,7 @@ export const excess = (codec: rt.RecordC): rt.InterfaceType either.chain(codec.validate(u, c), (o: Record) => { if (codecProps == null) { - return rt.failure( - u, - c, - `Invalid Aggs object ${JSON.stringify(u)} supplied to : ${codec.name}` - ); + return rt.failure(u, c, `Invalid Aggs object ${JSON.stringify(u)}`); } const keys = Object.keys(o); const ex = keys.reduce((acc, k) => { @@ -108,9 +104,7 @@ export const excess = (codec: rt.RecordC): rt.InterfaceType Date: Mon, 6 Jul 2020 21:43:31 -0400 Subject: [PATCH 09/52] doc --- ...bana-plugin-core-public.savedobjectsclient.find.md | 2 +- .../kibana-plugin-core-public.savedobjectsclient.md | 2 +- ...plugin-core-public.savedobjectsfindoptions.aggs.md | 11 +++++++++++ ...bana-plugin-core-public.savedobjectsfindoptions.md | 1 + ...lic.savedobjectsfindresponsepublic.aggregations.md | 11 +++++++++++ ...ugin-core-public.savedobjectsfindresponsepublic.md | 3 ++- ...plugin-core-server.savedobjectsfindoptions.aggs.md | 2 +- ...bana-plugin-core-server.savedobjectsfindoptions.md | 2 +- ...-plugin-core-server.savedobjectsrepository.find.md | 2 +- src/core/public/public.api.md | 11 +++++++++-- .../saved_objects/service/saved_objects_client.ts | 2 +- src/core/server/server.api.md | 7 +++++-- src/plugins/data/public/public.api.md | 1 + src/plugins/data/server/server.api.md | 1 + 14 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md index ddd8b207e3d785..fc9652b96450ff 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md @@ -9,5 +9,5 @@ Search for objects Signature: ```typescript -find: (options: SavedObjectsFindOptions) => Promise>; +find: (options: SavedObjectsFindOptions) => Promise>; ``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md index 904b9cce09d4e8..c93027c48c1b7f 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md @@ -24,7 +24,7 @@ The constructor for this class is marked as internal. Third-party code should no | [bulkGet](./kibana-plugin-core-public.savedobjectsclient.bulkget.md) | | (objects?: Array<{
id: string;
type: string;
}>) => Promise<SavedObjectsBatchResponse<unknown>> | Returns an array of objects by id | | [create](./kibana-plugin-core-public.savedobjectsclient.create.md) | | <T = unknown>(type: string, attributes: T, options?: SavedObjectsCreateOptions) => Promise<SimpleSavedObject<T>> | Persists an object | | [delete](./kibana-plugin-core-public.savedobjectsclient.delete.md) | | (type: string, id: string) => ReturnType<SavedObjectsApi['delete']> | Deletes an object | -| [find](./kibana-plugin-core-public.savedobjectsclient.find.md) | | <T = unknown>(options: SavedObjectsFindOptions) => Promise<SavedObjectsFindResponsePublic<T>> | Search for objects | +| [find](./kibana-plugin-core-public.savedobjectsclient.find.md) | | <T = unknown, A = unknown>(options: SavedObjectsFindOptions) => Promise<SavedObjectsFindResponsePublic<T, unknown>> | Search for objects | | [get](./kibana-plugin-core-public.savedobjectsclient.get.md) | | <T = unknown>(type: string, id: string) => Promise<SimpleSavedObject<T>> | Fetches a single object | ## Methods diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md new file mode 100644 index 00000000000000..5509baeb428ec3 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindOptions](./kibana-plugin-core-public.savedobjectsfindoptions.md) > [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) + +## SavedObjectsFindOptions.aggs property + +Signature: + +```typescript +aggs?: SavedObjectAggs; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md index 5f33d62382818a..7bd22af761868a 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md @@ -15,6 +15,7 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions | Property | Type | Description | | --- | --- | --- | +| [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) | SavedObjectAggs | | | [defaultSearchOperator](./kibana-plugin-core-public.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | | | [fields](./kibana-plugin-core-public.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-public.savedobjectsfindoptions.filter.md) | string | | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md new file mode 100644 index 00000000000000..14401b02f25c74 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindResponsePublic](./kibana-plugin-core-public.savedobjectsfindresponsepublic.md) > [aggregations](./kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md) + +## SavedObjectsFindResponsePublic.aggregations property + +Signature: + +```typescript +aggregations?: A; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md index 7d75878041264d..6f2276194f054f 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md @@ -11,13 +11,14 @@ Return type of the Saved Objects `find()` method. Signature: ```typescript -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse +export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse ``` ## Properties | Property | Type | Description | | --- | --- | --- | +| [aggregations](./kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md) | A | | | [page](./kibana-plugin-core-public.savedobjectsfindresponsepublic.page.md) | number | | | [perPage](./kibana-plugin-core-public.savedobjectsfindresponsepublic.perpage.md) | number | | | [total](./kibana-plugin-core-public.savedobjectsfindresponsepublic.total.md) | number | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md index 46b4936295c508..6ad9fe06ef1fc5 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md @@ -7,5 +7,5 @@ Signature: ```typescript -aggs?: string; +aggs?: SavedObjectAggs; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index 5b76cbae6a5a00..c9f4d513242950 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -15,7 +15,7 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | string | | +| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | SavedObjectAggs | | | [defaultSearchOperator](./kibana-plugin-core-server.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | | | [fields](./kibana-plugin-core-server.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md index bcbeb6d44d6c88..47d9fd79e4802a 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md @@ -7,7 +7,7 @@ Signature: ```typescript -find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, preference, aggs, }: SavedObjectsFindOptions): Promise>; +find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, preference, aggs, }: SavedObjectsFindOptions): Promise>; ``` ## Parameters diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 86e281a49b744a..273443e99599af 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -120,6 +120,7 @@ import { RecursiveReadonly } from '@kbn/utility-types'; import { ReindexParams } from 'elasticsearch'; import { ReindexRethrottleParams } from 'elasticsearch'; import { RenderSearchTemplateParams } from 'elasticsearch'; +import * as rt from 'io-ts'; import * as Rx from 'rxjs'; import { ScrollParams } from 'elasticsearch'; import { SearchParams } from 'elasticsearch'; @@ -1258,7 +1259,7 @@ export class SavedObjectsClient { // Warning: (ae-forgotten-export) The symbol "SavedObjectsClientContract" needs to be exported by the entry point index.d.ts delete: (type: string, id: string) => ReturnType; // Warning: (ae-forgotten-export) The symbol "SavedObjectsFindOptions" needs to be exported by the entry point index.d.ts - find: (options: SavedObjectsFindOptions_2) => Promise>; + find: (options: SavedObjectsFindOptions_2) => Promise>; get: (type: string, id: string) => Promise>; update(type: string, id: string, attributes: T, { version, migrationVersion, references }?: SavedObjectsUpdateOptions): Promise>; } @@ -1277,6 +1278,10 @@ export interface SavedObjectsCreateOptions { // @public (undocumented) export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { + // Warning: (ae-forgotten-export) The symbol "SavedObjectAggs" needs to be exported by the entry point index.d.ts + // + // (undocumented) + aggs?: SavedObjectAggs; // (undocumented) defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; @@ -1303,7 +1308,9 @@ export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { } // @public -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { +export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { + // (undocumented) + aggregations?: A; // (undocumented) page: number; // (undocumented) diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index db51102e0d656a..16f3643893d830 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -100,7 +100,7 @@ export interface SavedObjectsFindResult extends SavedObject { */ export interface SavedObjectsFindResponse { aggregations?: A; - saved_objects: Array>; + saved_objects: Array>; total: number; per_page: number; page: number; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 4dd5a06198ea47..eaa0e0207f0936 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -115,6 +115,7 @@ import { RenderSearchTemplateParams } from 'elasticsearch'; import { Request } from 'hapi'; import { ResponseObject } from 'hapi'; import { ResponseToolkit } from 'hapi'; +import * as rt from 'io-ts'; import { SchemaTypeError } from '@kbn/config-schema'; import { ScrollParams } from 'elasticsearch'; import { SearchParams } from 'elasticsearch'; @@ -2114,8 +2115,10 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions extends SavedObjectsBaseOptions { + // Warning: (ae-forgotten-export) The symbol "SavedObjectAggs" needs to be exported by the entry point index.d.ts + // // (undocumented) - aggs?: string; + aggs?: SavedObjectAggs; // (undocumented) defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; @@ -2340,7 +2343,7 @@ export class SavedObjectsRepository { deleteByNamespace(namespace: string, options?: SavedObjectsDeleteByNamespaceOptions): Promise; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise<{}>; // (undocumented) - find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, preference, aggs, }: SavedObjectsFindOptions): Promise>; + find({ search, defaultSearchOperator, searchFields, hasReference, page, perPage, sortField, sortOrder, fields, namespace, type, filter, preference, aggs, }: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFieldName: string, options?: SavedObjectsIncrementCounterOptions): Promise<{ id: string; diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 670b40e7d94722..7cba6386c790f8 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -145,6 +145,7 @@ import { ReindexParams } from 'elasticsearch'; import { ReindexRethrottleParams } from 'elasticsearch'; import { RenderSearchTemplateParams } from 'elasticsearch'; import { Required } from '@kbn/utility-types'; +import * as rt from 'io-ts'; import * as Rx from 'rxjs'; import { SavedObject } from 'src/core/server'; import { SavedObject as SavedObject_3 } from 'src/core/public'; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index f029609cbf7ec5..00ab8e500849cf 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -118,6 +118,7 @@ import { RenderSearchTemplateParams } from 'elasticsearch'; import { Request } from 'hapi'; import { ResponseObject } from 'hapi'; import { ResponseToolkit } from 'hapi'; +import * as rt from 'io-ts'; import { SavedObject as SavedObject_2 } from 'src/core/server'; import { SchemaTypeError } from '@kbn/config-schema'; import { ScrollParams } from 'elasticsearch'; From e698b0ef8abda43c671bfafbc4fb296529439a55 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Tue, 7 Jul 2020 11:10:09 -0400 Subject: [PATCH 10/52] try to fix test --- .../saved_objects/spaces/mappings.json | 1 + .../mappings.json | 18 ------------------ 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index 2c1023b2854b65..54a17ad88012a2 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -81,6 +81,7 @@ } }, "globaltype": { + "dynamic": "true", "properties": { "title": { "fields": { diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json deleted file mode 100644 index f2b4b58369e1a9..00000000000000 --- a/x-pack/test/saved_object_api_integration/common/fixtures/namespace_agnostic_type_plugin/mappings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "globaltype": { - "properties": { - "title": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 2048 - } - } - }, - "version": { - "type": "integer" - } - } - } -} From c16e72b9ac69cb8619e03465f732bbe64b9e8b27 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Tue, 7 Jul 2020 11:40:54 -0400 Subject: [PATCH 11/52] add the new property to the saved object globaltype --- .../fixtures/saved_object_test_plugin/server/plugin.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts b/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts index 0c15ab4bd2f804..503349ba6e1741 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts +++ b/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts @@ -55,7 +55,14 @@ export class Plugin { hidden: false, namespaceType: 'agnostic', management, - mappings, + mappings: { + properties: { + ...mappings.properties, + version: { + type: 'integer', + }, + }, + }, }); core.savedObjects.registerType({ name: 'hiddentype', From 5a12d44ed626a7a4b7dcb7dfd3255f881e4547f0 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Tue, 7 Jul 2020 15:14:47 -0400 Subject: [PATCH 12/52] fix types --- .../saved_objects/encrypted_saved_objects_client_wrapper.ts | 4 ++-- .../saved_objects/secure_saved_objects_client_wrapper.ts | 4 ++-- .../server/saved_objects/spaces_saved_objects_client.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index bdc2b6cb2e6679..c560aab23cc09f 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -165,9 +165,9 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.delete(type, id, options); } - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { return await this.handleEncryptedAttributesInBulkResponse( - await this.options.baseClient.find(options), + await this.options.baseClient.find(options), undefined, options.namespace ); diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index 969344afae5e37..23ca0ea81a3138 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -98,10 +98,10 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return await this.baseClient.delete(type, id, options); } - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { await this.ensureAuthorized(options.type, 'find', options.namespace, { options }); - const response = await this.baseClient.find(options); + const response = await this.baseClient.find(options); return await this.redactSavedObjectsNamespaces(response); } diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index 6611725be8b67b..46dd48d9151e37 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -135,10 +135,10 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * @property {object} [options.hasReference] - { type, id } * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { throwErrorIfNamespaceSpecified(options); - return await this.client.find({ + return await this.client.find({ ...options, type: (options.type ? coerceToArray(options.type) : this.types).filter( (type) => type !== 'space' From f04ab64181448951ca7c3db895f87f3224569827 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:28:02 -0500 Subject: [PATCH 13/52] delete old files --- x-pack/plugins/security_solution/index.scss | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 x-pack/plugins/security_solution/index.scss diff --git a/x-pack/plugins/security_solution/index.scss b/x-pack/plugins/security_solution/index.scss deleted file mode 100644 index 8b2f6d3cb61563..00000000000000 --- a/x-pack/plugins/security_solution/index.scss +++ /dev/null @@ -1,17 +0,0 @@ -/* GIS plugin styles */ - -// Import the EUI global scope so we can use EUI constants -@import 'src/legacy/ui/public/styles/_styling_constants'; - -// Prefix all styles with "map" to avoid conflicts. -// Examples -// mapChart -// mapChart__legend -// mapChart__legend--small -// mapChart__legend-isLoading - -@import 'main'; -@import 'mapbox_hacks'; -@import 'connected_components/index'; -@import 'components/index'; -@import 'layers/index'; From 8f64144cf73bf1e88514d299d358af71713a0b25 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 9 Nov 2020 16:29:12 -0500 Subject: [PATCH 14/52] fix types + test api integration --- .../spaces_saved_objects_client.ts | 4 ++-- .../common/suites/find.ts | 22 ++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index cda3c48e71ccb8..08e18524d67461 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -180,12 +180,12 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { } if (namespaces.length === 0) { // return empty response, since the user is unauthorized in this space (or these spaces), but we don't return forbidden errors for `find` operations - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } } catch (err) { if (Boom.isBoom(err) && err.output.payload.statusCode === 403) { // return empty response, since the user is unauthorized in any space, but we don't return forbidden errors for `find` operations - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } throw err; } diff --git a/x-pack/test/saved_object_api_integration/common/suites/find.ts b/x-pack/test/saved_object_api_integration/common/suites/find.ts index 88cb9fce38615e..9fbd1458a15304 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/find.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/find.ts @@ -170,40 +170,40 @@ export const getTestCases = ( typeUseInQueryField: 'dashboard', }, aggsWithNamespaceAgnosticType: { - title: 'aggs with namespace-agnostic type', + title: buildTitle('aggs with namespace-agnostic type'), query: `type=globaltype&aggs=${encodeURIComponent( JSON.stringify({ type_count: { max: { field: 'globaltype.attributes.version' } }, }) - )}`, + )}${namespacesQueryParam}`, successResult: { savedObjects: SAVED_OBJECT_TEST_CASES.NAMESPACE_AGNOSTIC }, typeUseInQueryField: 'globaltype', }, aggsWithHiddenType: { - title: 'aggs with hidden type', + title: buildTitle('aggs with hidden type'), query: `type=hiddentype&fields=name&aggs=${encodeURIComponent( JSON.stringify({ type_count: { max: { field: 'hiddentype.attributes.title' } }, }) - )}`, + )}${namespacesQueryParam}`, typeUseInQueryField: 'hiddentype', }, aggsWithUnknownType: { - title: 'aggs with unknown type', + title: buildTitle('aggs with unknown type'), query: `type=wigwags&aggs=${encodeURIComponent( JSON.stringify({ type_count: { max: { field: 'wigwags.attributes.version' } }, }) - )}`, + )}${namespacesQueryParam}`, typeUseInQueryField: 'wigwags', }, aggsWithDisallowedType: { - title: 'aggs with disallowed type', + title: buildTitle('aggs with disallowed type'), query: `type=globaltype&aggs=${encodeURIComponent( JSON.stringify({ type_count: { max: { field: 'dashboard.attributes.version' } }, }) - )}`, + )}${namespacesQueryParam}`, failure: { statusCode: 400, reason: 'bad_request', @@ -222,7 +222,6 @@ const getTestTitle = ({ failure, title }: FindTestCase) => `${failure?.reason || 'success'} ["${title}"]`; export function findTestSuiteFactory(esArchiver: any, supertest: SuperTest) { - const expectForbidden = expectResponses.forbiddenTypes('find'); const expectResponseBody = ( testCase: FindTestCase, user?: TestUser @@ -242,12 +241,9 @@ export function findTestSuiteFactory(esArchiver: any, supertest: SuperTest) } else { throw new Error(`Unexpected failure reason: ${failure.reason}`); } - } else if (failure?.statusCode === 403) { - const type = parsedQuery.type; - await expectForbidden(type)(response); } else if (failure?.statusCode === 400) { if (failure.reason === 'bad_request') { - const type = (parsedQuery.filter as string).split('.')[0]; + const type = testCase.typeUseInQueryField ?? 'unknown type'; expect(response.body.error).to.eql('Bad Request'); expect(response.body.statusCode).to.eql(failure.statusCode); expect(response.body.message).to.eql(`This type ${type} is not allowed: Bad Request`); From 2209d12f1456e4b477a738fd7c1f5bbcc7a2d5d6 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Tue, 10 Nov 2020 16:12:39 -0500 Subject: [PATCH 15/52] type fix + test --- test/api_integration/apis/saved_objects/find.js | 2 +- x-pack/test/saved_object_api_integration/common/suites/find.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/api_integration/apis/saved_objects/find.js b/test/api_integration/apis/saved_objects/find.js index 075abd9a14409c..0ea95448f87b9b 100644 --- a/test/api_integration/apis/saved_objects/find.js +++ b/test/api_integration/apis/saved_objects/find.js @@ -359,7 +359,7 @@ export default function ({ getService }) { expect(resp.body).to.eql({ error: 'Bad Request', message: - 'Invalid value {"type_count":{"max":{"field":"dashboard.attributes.version","script":"Oh yes I am going to a script"}}} supplied to : { [K in string]: ((Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) & Partial<{ aggs: (Partial<{ filter: { term: { [K in string]: string } }, histogram: ({ field: string } & { interval: number } & Partial<{ min_doc_count: number, extended_bounds: { min: number, max: number }, keyed: boolean, missing: number, order: { [K in string]: desc } }>), terms: ({ field: string } & Partial<{ field: string, size: number, show_term_doc_count_error: boolean, order: { [K in string]: desc } }>) }> & Partial<{ avg: { field: string }, weighted_avg: ({ value: ({ field: string } & Partial<{ missing: number }>), weight: ({ field: string } & Partial<{ missing: number }>) } & Partial<{ format: string, value_type: string }>), cardinality: { field: string }, max: ({ field: string } & Partial<{ missing: number }>), min: ({ field: string } & Partial<{ missing: number }>), top_hits: Partial<{ explain: boolean, from: string, highlight: any, seq_no_primary_term: boolean, size: number, sort: any, stored_fields: Array, version: boolean, _name: string, _source: Partial<{ includes: Array, excludes: Array }> }> }>) }>) }, excess properties: ["script"]: Bad Request', + 'Invalid value {"type_count":{"max":{"field":"dashboard.attributes.version","script":"Oh yes I am going to a script"}}}, excess properties: ["script"]: Bad Request', statusCode: 400, }); })); diff --git a/x-pack/test/saved_object_api_integration/common/suites/find.ts b/x-pack/test/saved_object_api_integration/common/suites/find.ts index 9fbd1458a15304..918d3c9c7faf35 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/find.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/find.ts @@ -13,7 +13,6 @@ import { getUrlPrefix, isUserAuthorizedAtSpace, getRedactedNamespaces, - expectResponses, } from '../lib/saved_object_test_utils'; import { ExpectResponseBody, TestCase, TestDefinition, TestSuite, TestUser } from '../lib/types'; From 4f667d48233a63281ec2e814f3cbbbd0f2e79573 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Wed, 11 Nov 2020 11:38:58 -0500 Subject: [PATCH 16/52] Update src/core/server/saved_objects/types.ts Co-authored-by: Rudolf Meijering --- src/core/server/saved_objects/types.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 5712c0d6fcab5f..dda5f5d323edd6 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -109,6 +109,11 @@ export interface SavedObjectsFindOptions { */ defaultSearchOperator?: 'AND' | 'OR'; filter?: string | KueryNode; + /** + * Specify an Elasticsearch aggregation to perform. This alpha API only supports a limited set of aggregation types: metrics, bucket. Additional aggregation types can be contributed to Core. + * @alpha + * @example // Can you add an code example of how to use the API that shows an aggregation on a root property and an type attribute + */ aggs?: SavedObjectAggs; namespaces?: string[]; /** From 433a71cbfc4a3ce95f8eaa8cb3f4a9d7ea0a5c6a Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Wed, 11 Nov 2020 15:54:57 -0500 Subject: [PATCH 17/52] review I --- .../core/public/kibana-plugin-core-public.md | 1 + ...ana-plugin-core-public.savedobjectsaggs.md | 11 +++++++++ ...n-core-public.savedobjectscreateoptions.md | 1 - ...ore-public.savedobjectsfindoptions.aggs.md | 2 +- ...gin-core-public.savedobjectsfindoptions.md | 2 +- .../core/server/kibana-plugin-core-server.md | 1 + ...ana-plugin-core-server.savedobjectsaggs.md | 11 +++++++++ ...ore-server.savedobjectsfindoptions.aggs.md | 2 +- ...gin-core-server.savedobjectsfindoptions.md | 2 +- src/core/public/index.ts | 1 + src/core/public/public.api.md | 12 +++++++--- src/core/public/saved_objects/index.ts | 1 + .../saved_objects/saved_objects_client.ts | 3 +++ src/core/server/index.ts | 1 + .../service/lib/aggs_utils.test.ts | 24 +++++++++---------- .../saved_objects/service/lib/aggs_utils.ts | 10 ++++---- .../server/saved_objects/service/lib/index.ts | 2 +- .../saved_objects/service/lib/repository.ts | 4 ++-- .../bucket_aggs/index.ts | 0 .../helpers.test.ts | 0 .../helpers.ts | 0 .../index.ts | 8 +++---- .../metrics_aggs/index.ts | 0 .../service/saved_objects_client.ts | 4 ++-- src/core/server/saved_objects/types.ts | 4 ++-- src/core/server/server.api.md | 10 +++++--- .../saved_objects/spaces/mappings.json | 1 - 27 files changed, 78 insertions(+), 40 deletions(-) create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md rename src/core/server/saved_objects/service/lib/{saved_object_aggs_types => saved_objects_aggs_types}/bucket_aggs/index.ts (100%) rename src/core/server/saved_objects/service/lib/{saved_object_aggs_types => saved_objects_aggs_types}/helpers.test.ts (100%) rename src/core/server/saved_objects/service/lib/{saved_object_aggs_types => saved_objects_aggs_types}/helpers.ts (100%) rename src/core/server/saved_objects/service/lib/{saved_object_aggs_types => saved_objects_aggs_types}/index.ts (89%) rename src/core/server/saved_objects/service/lib/{saved_object_aggs_types => saved_objects_aggs_types}/metrics_aggs/index.ts (100%) diff --git a/docs/development/core/public/kibana-plugin-core-public.md b/docs/development/core/public/kibana-plugin-core-public.md index b8b1bdcdee3be3..9a9876682c7027 100644 --- a/docs/development/core/public/kibana-plugin-core-public.md +++ b/docs/development/core/public/kibana-plugin-core-public.md @@ -158,6 +158,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [PublicUiSettingsParams](./kibana-plugin-core-public.publicuisettingsparams.md) | A sub-set of [UiSettingsParams](./kibana-plugin-core-public.uisettingsparams.md) exposed to the client-side. | | [SavedObjectAttribute](./kibana-plugin-core-public.savedobjectattribute.md) | Type definition for a Saved Object attribute value | | [SavedObjectAttributeSingle](./kibana-plugin-core-public.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-public.savedobjectattribute.md) | +| [SavedObjectsAggs](./kibana-plugin-core-public.savedobjectsaggs.md) | | | [SavedObjectsClientContract](./kibana-plugin-core-public.savedobjectsclientcontract.md) | SavedObjectsClientContract as implemented by the [SavedObjectsClient](./kibana-plugin-core-public.savedobjectsclient.md) | | [SavedObjectsNamespaceType](./kibana-plugin-core-public.savedobjectsnamespacetype.md) | The namespace type dictates how a saved object can be interacted in relation to namespaces. Each type is mutually exclusive: \* single (default): this type of saved object is namespace-isolated, e.g., it exists in only one namespace. \* multiple: this type of saved object is shareable, e.g., it can exist in one or more namespaces. \* agnostic: this type of saved object is global. | | [StartServicesAccessor](./kibana-plugin-core-public.startservicesaccessor.md) | Allows plugins to get access to APIs available in start inside async handlers, such as [App.mount](./kibana-plugin-core-public.app.mount.md). Promise will not resolve until Core and plugin dependencies have completed start. | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md new file mode 100644 index 00000000000000..2626086355a98c --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsAggs](./kibana-plugin-core-public.savedobjectsaggs.md) + +## SavedObjectsAggs type + +Signature: + +```typescript +export declare type SavedObjectsAggs = rt.TypeOf; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md index b1b93407d4ff10..85a34708cc0dc8 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md @@ -4,7 +4,6 @@ ## SavedObjectsCreateOptions interface - Signature: ```typescript diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md index 5509baeb428ec3..f09d7922e28b97 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md @@ -7,5 +7,5 @@ Signature: ```typescript -aggs?: SavedObjectAggs; +aggs?: SavedObjectsAggs; ``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md index 29b77d5a19e54d..80b3b2b5cbbfc9 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md @@ -15,7 +15,7 @@ export interface SavedObjectsFindOptions | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) | SavedObjectAggs | | +| [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) | SavedObjectsAggs | | | [defaultSearchOperator](./kibana-plugin-core-public.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | The search operator to use with the provided filter. Defaults to OR | | [fields](./kibana-plugin-core-public.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-public.savedobjectsfindoptions.filter.md) | string | KueryNode | | diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 68f5e729155567..55a5cc86b77053 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -284,6 +284,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | Type definition for a Saved Object attribute value | | [SavedObjectAttributeSingle](./kibana-plugin-core-server.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | | [SavedObjectMigrationFn](./kibana-plugin-core-server.savedobjectmigrationfn.md) | A migration function for a [saved object type](./kibana-plugin-core-server.savedobjectstype.md) used to migrate it to a given version | +| [SavedObjectsAggs](./kibana-plugin-core-server.savedobjectsaggs.md) | | | [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) | Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a references root property. | | [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.\#\# SavedObjectsClient errorsSince the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the isXYZError() helpers exposed at SavedObjectsErrorHelpers should be used to understand and manage error responses from the SavedObjectsClient.Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for error.body.error.type or doing substring checks on error.body.error.reason, just use the helpers to understand the meaning of the error:\`\`\`js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }// always rethrow the error unless you handle it throw error; \`\`\`\#\#\# 404s from missing indexFrom the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) | | [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md new file mode 100644 index 00000000000000..84f9e138c5408c --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsAggs](./kibana-plugin-core-server.savedobjectsaggs.md) + +## SavedObjectsAggs type + +Signature: + +```typescript +export declare type SavedObjectsAggs = rt.TypeOf; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md index 6ad9fe06ef1fc5..25588f623656a1 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md @@ -7,5 +7,5 @@ Signature: ```typescript -aggs?: SavedObjectAggs; +aggs?: SavedObjectsAggs; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index ab1a73962c2110..3587d57069124e 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -15,7 +15,7 @@ export interface SavedObjectsFindOptions | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | SavedObjectAggs | | +| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | SavedObjectsAggs | | | [defaultSearchOperator](./kibana-plugin-core-server.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | The search operator to use with the provided filter. Defaults to OR | | [fields](./kibana-plugin-core-server.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | string | KueryNode | | diff --git a/src/core/public/index.ts b/src/core/public/index.ts index 1393e69d55e512..b90bf20406d2c7 100644 --- a/src/core/public/index.ts +++ b/src/core/public/index.ts @@ -115,6 +115,7 @@ export { } from './application'; export { + SavedObjectsAggs, SavedObjectsBatchResponse, SavedObjectsBulkCreateObject, SavedObjectsBulkCreateOptions, diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 3d84dac2a70299..e284c47460df6e 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -980,6 +980,12 @@ export interface SavedObjectReference { type: string; } +// Warning: (ae-forgotten-export) The symbol "SavedObjectsAggsRt" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "SavedObjectsAggs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type SavedObjectsAggs = rt.TypeOf; + // @public (undocumented) export interface SavedObjectsBaseOptions { namespace?: string; @@ -1047,6 +1053,8 @@ export class SavedObjectsClient { // @public export type SavedObjectsClientContract = PublicMethodsOf; +// Warning: (ae-missing-release-tag) "SavedObjectsCreateOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// // @public (undocumented) export interface SavedObjectsCreateOptions { id?: string; @@ -1058,10 +1066,8 @@ export interface SavedObjectsCreateOptions { // @public (undocumented) export interface SavedObjectsFindOptions { - // Warning: (ae-forgotten-export) The symbol "SavedObjectAggs" needs to be exported by the entry point index.d.ts - // // (undocumented) - aggs?: SavedObjectAggs; + aggs?: SavedObjectsAggs; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts diff --git a/src/core/public/saved_objects/index.ts b/src/core/public/saved_objects/index.ts index cc8fce0884ddf9..06f66fdad24839 100644 --- a/src/core/public/saved_objects/index.ts +++ b/src/core/public/saved_objects/index.ts @@ -18,6 +18,7 @@ */ export { + SavedObjectsAggs, SavedObjectsBatchResponse, SavedObjectsBulkCreateObject, SavedObjectsBulkCreateOptions, diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 0ac4c6ab63af58..98b228150012e1 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -24,6 +24,7 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { SavedObject, SavedObjectReference, + SavedObjectsAggs, SavedObjectsClientContract as SavedObjectsApi, SavedObjectsFindOptions as SavedObjectFindOptionsServer, SavedObjectsMigrationVersion, @@ -40,6 +41,8 @@ type SavedObjectsFindOptions = Omit< type PromiseType> = T extends Promise ? U : never; /** @public */ +export { SavedObjectsAggs }; + export interface SavedObjectsCreateOptions { /** * (Not recommended) Specify an id instead of having the saved objects service generate one for you. diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 0adda4770639d7..3585820c657b3c 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -240,6 +240,7 @@ export { } from './plugins'; export { + SavedObjectsAggs, SavedObjectsBulkCreateObject, SavedObjectsBulkGetObject, SavedObjectsBulkUpdateObject, diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index fcc32d92b63634..3b9bb6332e8b9c 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -17,15 +17,15 @@ * under the License. */ -import { validateGetSavedObjectAggs } from './aggs_utils'; +import { validateGetSavedObjectsAggs } from './aggs_utils'; import { mockMappings } from './filter_utils.test'; -import { SavedObjectAggs } from './saved_object_aggs_types'; +import { SavedObjectsAggs } from './saved_objects_aggs_types'; describe('Filter Utils', () => { - describe('#validateGetSavedObjectAggs', () => { + describe('#validateGetSavedObjectsAggs', () => { test('Validate a simple aggregations', () => { expect( - validateGetSavedObjectAggs( + validateGetSavedObjectsAggs( ['foo'], { aggName: { max: { field: 'foo.attributes.bytes' } } }, mockMappings @@ -40,7 +40,7 @@ describe('Filter Utils', () => { }); test('Validate a nested simple aggregations', () => { expect( - validateGetSavedObjectAggs( + validateGetSavedObjectsAggs( ['alert'], { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, mockMappings @@ -56,7 +56,7 @@ describe('Filter Utils', () => { test('Throw an error when types is not allowed', () => { expect(() => { - validateGetSavedObjectAggs( + validateGetSavedObjectsAggs( ['alert'], { aggName: { @@ -68,15 +68,15 @@ describe('Filter Utils', () => { }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed: Bad Request"`); }); - test('Throw an error when aggregation is not defined in SavedObjectAggs', () => { + test('Throw an error when aggregation is not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectAggs( + validateGetSavedObjectsAggs( ['foo'], { aggName: { MySuperAgg: { field: 'foo.attributes.bytes' }, }, - } as SavedObjectAggs, + } as SavedObjectsAggs, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( @@ -84,16 +84,16 @@ describe('Filter Utils', () => { ); }); - test('Throw an error when you add attributes who are not defined in SavedObjectAggs', () => { + test('Throw an error when you add attributes who are not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectAggs( + validateGetSavedObjectsAggs( ['alert'], { aggName: { cardinality: { field: 'alert.attributes.actions.group' }, script: 'I want to access that I should not', }, - } as SavedObjectAggs, + } as SavedObjectsAggs, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index b3850498ee9584..b88ce5b50dc8bd 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -21,13 +21,13 @@ import { IndexMapping } from '../../mappings'; import { SavedObjectsErrorHelpers } from './errors'; import { hasFilterKeyError } from './filter_utils'; -import { SavedObjectAggs, validateSavedObjectTypeAggs } from './saved_object_aggs_types'; +import { SavedObjectsAggs, validateSavedObjectTypeAggs } from './saved_objects_aggs_types'; -export const validateGetSavedObjectAggs = ( +export const validateGetSavedObjectsAggs = ( allowedTypes: string[], - aggs: SavedObjectAggs, + aggs: SavedObjectsAggs, indexMapping: IndexMapping -): SavedObjectAggs => { +): SavedObjectsAggs => { validateSavedObjectTypeAggs(aggs); return validateGetAggFieldValue(allowedTypes, aggs, indexMapping); }; @@ -36,7 +36,7 @@ const validateGetAggFieldValue = ( allowedTypes: string[], aggs: any, indexMapping: IndexMapping -): SavedObjectAggs => { +): SavedObjectsAggs => { return Object.keys(aggs).reduce((acc, key) => { if (key === 'field') { const error = hasFilterKeyError(aggs[key], allowedTypes, indexMapping); diff --git a/src/core/server/saved_objects/service/lib/index.ts b/src/core/server/saved_objects/service/lib/index.ts index 70a279b930a211..8972c57b2464aa 100644 --- a/src/core/server/saved_objects/service/lib/index.ts +++ b/src/core/server/saved_objects/service/lib/index.ts @@ -31,5 +31,5 @@ export { export { SavedObjectsErrorHelpers } from './errors'; -export { SavedObjectAggs } from './saved_object_aggs_types'; +export { SavedObjectsAggs } from './saved_objects_aggs_types'; export { SavedObjectsUtils } from './utils'; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index e381e990d0463b..830b9cdd1136ae 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -69,7 +69,7 @@ import { } from '../../types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; -import { validateGetSavedObjectAggs } from './aggs_utils'; +import { validateGetSavedObjectsAggs } from './aggs_utils'; import { ALL_NAMESPACES_STRING, FIND_DEFAULT_PAGE, @@ -781,7 +781,7 @@ export class SavedObjectsRepository { let aggsObject = null; try { if (aggs) { - aggsObject = validateGetSavedObjectAggs(allowedTypes, aggs, this._mappings); + aggsObject = validateGetSavedObjectsAggs(allowedTypes, aggs, this._mappings); } } catch (e) { throw e; diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts similarity index 100% rename from src/core/server/saved_objects/service/lib/saved_object_aggs_types/bucket_aggs/index.ts rename to src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts similarity index 100% rename from src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.test.ts rename to src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts similarity index 100% rename from src/core/server/saved_objects/service/lib/saved_object_aggs_types/helpers.ts rename to src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts similarity index 89% rename from src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts rename to src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts index 09c6d4cb204e69..afe8e3b8b4a18e 100644 --- a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts @@ -31,16 +31,16 @@ import { excess, throwErrors } from './helpers'; const AllAggsRt = rt.intersection([BucketAggsTypeRt, MetricsAggsTypeRt]); -const SavedObjectAggsRt = rt.record( +const SavedObjectsAggsRt = rt.record( rt.string, rt.intersection([AllAggsRt, rt.partial({ aggs: AllAggsRt })]) ); -export type SavedObjectAggs = rt.TypeOf; +export type SavedObjectsAggs = rt.TypeOf; -export const validateSavedObjectTypeAggs = (aggObjects: SavedObjectAggs) => { +export const validateSavedObjectTypeAggs = (aggObjects: SavedObjectsAggs) => { pipe( - excess(SavedObjectAggsRt).decode(aggObjects), + excess(SavedObjectsAggsRt).decode(aggObjects), fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) ); }; diff --git a/src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts similarity index 100% rename from src/core/server/saved_objects/service/lib/saved_object_aggs_types/metrics_aggs/index.ts rename to src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 14db8adabc6e54..d6ef6a29a7b066 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ISavedObjectsRepository, SavedObjectAggs } from './lib'; +import { ISavedObjectsRepository, SavedObjectsAggs } from './lib'; import { SavedObject, SavedObjectError, @@ -289,7 +289,7 @@ export interface SavedObjectsUpdateResponse * * @public */ -export { SavedObjectAggs }; +export { SavedObjectsAggs }; /** * diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 5712c0d6fcab5f..91bf2b5d64fad6 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -17,7 +17,7 @@ * under the License. */ -import { SavedObjectsClient, SavedObjectAggs } from './service/saved_objects_client'; +import { SavedObjectsClient, SavedObjectsAggs } from './service/saved_objects_client'; import { SavedObjectsTypeMappingDefinition } from './mappings'; import { SavedObjectMigrationMap } from './migrations'; @@ -109,7 +109,7 @@ export interface SavedObjectsFindOptions { */ defaultSearchOperator?: 'AND' | 'OR'; filter?: string | KueryNode; - aggs?: SavedObjectAggs; + aggs?: SavedObjectsAggs; namespaces?: string[]; /** * This map defines each type to search for, and the namespace(s) to search for the type in; this is only intended to be used by a saved diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index da2ac0a7873b11..684920a678a2ef 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1884,6 +1884,12 @@ export interface SavedObjectsAddToNamespacesResponse { namespaces: string[]; } +// Warning: (ae-forgotten-export) The symbol "SavedObjectsAggsRt" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "SavedObjectsAggs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type SavedObjectsAggs = rt.TypeOf; + // Warning: (ae-forgotten-export) The symbol "SavedObjectDoc" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "Referencable" needs to be exported by the entry point index.d.ts // @@ -2180,10 +2186,8 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions { - // Warning: (ae-forgotten-export) The symbol "SavedObjectAggs" needs to be exported by the entry point index.d.ts - // // (undocumented) - aggs?: SavedObjectAggs; + aggs?: SavedObjectsAggs; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index b89d7aa6b80e39..04fed1905ecc5b 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -81,7 +81,6 @@ } }, "globaltype": { - "dynamic": "true", "properties": { "title": { "fields": { From b23eac7fc7cd97865f7835d61bb79a85817a0b3e Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Sun, 22 Nov 2020 22:04:54 -0500 Subject: [PATCH 18/52] change our validation to match discussion with Pierre and Rudolph --- .../core/public/kibana-plugin-core-public.md | 1 - ...ana-plugin-core-public.savedobjectsaggs.md | 11 - ...ore-public.savedobjectsfindoptions.aggs.md | 11 - ...gin-core-public.savedobjectsfindoptions.md | 1 - .../core/server/kibana-plugin-core-server.md | 1 - ...ana-plugin-core-server.savedobjectsaggs.md | 11 - ...ore-server.savedobjectsfindoptions.aggs.md | 11 - ...gin-core-server.savedobjectsfindoptions.md | 1 - src/core/public/index.ts | 1 - src/core/public/public.api.md | 11 +- src/core/public/saved_objects/index.ts | 1 - .../saved_objects/saved_objects_client.ts | 10 +- src/core/server/index.ts | 1 - src/core/server/saved_objects/routes/find.ts | 2 +- .../service/lib/aggs_utils.test.ts | 161 +++++++++++++- .../saved_objects/service/lib/aggs_utils.ts | 74 +++++-- .../server/saved_objects/service/lib/index.ts | 1 - .../saved_objects/service/lib/repository.ts | 25 ++- .../bucket_aggs/index.ts | 53 ++--- .../saved_objects_aggs_types/helpers.test.ts | 196 ------------------ .../lib/saved_objects_aggs_types/helpers.ts | 88 +------- .../lib/saved_objects_aggs_types/index.ts | 22 +- .../metrics_aggs/index.ts | 42 ++-- .../service/saved_objects_client.ts | 8 +- src/core/server/saved_objects/types.ts | 4 +- src/core/server/server.api.md | 11 +- src/plugins/data/public/public.api.md | 1 - src/plugins/embeddable/public/public.api.md | 1 - test/api_integration/apis/index.js | 28 +-- .../apis/saved_objects/find.js | 4 +- .../apis/saved_objects/index.js | 22 +- 31 files changed, 323 insertions(+), 492 deletions(-) delete mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md delete mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md delete mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md delete mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md delete mode 100644 src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts diff --git a/docs/development/core/public/kibana-plugin-core-public.md b/docs/development/core/public/kibana-plugin-core-public.md index f2cf0f5f811dd7..6a90fd49f1d667 100644 --- a/docs/development/core/public/kibana-plugin-core-public.md +++ b/docs/development/core/public/kibana-plugin-core-public.md @@ -163,7 +163,6 @@ The plugin integrates with the core system via lifecycle events: `setup` | [PublicUiSettingsParams](./kibana-plugin-core-public.publicuisettingsparams.md) | A sub-set of [UiSettingsParams](./kibana-plugin-core-public.uisettingsparams.md) exposed to the client-side. | | [SavedObjectAttribute](./kibana-plugin-core-public.savedobjectattribute.md) | Type definition for a Saved Object attribute value | | [SavedObjectAttributeSingle](./kibana-plugin-core-public.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-public.savedobjectattribute.md) | -| [SavedObjectsAggs](./kibana-plugin-core-public.savedobjectsaggs.md) | | | [SavedObjectsClientContract](./kibana-plugin-core-public.savedobjectsclientcontract.md) | SavedObjectsClientContract as implemented by the [SavedObjectsClient](./kibana-plugin-core-public.savedobjectsclient.md) | | [SavedObjectsNamespaceType](./kibana-plugin-core-public.savedobjectsnamespacetype.md) | The namespace type dictates how a saved object can be interacted in relation to namespaces. Each type is mutually exclusive: \* single (default): this type of saved object is namespace-isolated, e.g., it exists in only one namespace. \* multiple: this type of saved object is shareable, e.g., it can exist in one or more namespaces. \* agnostic: this type of saved object is global. | | [StartServicesAccessor](./kibana-plugin-core-public.startservicesaccessor.md) | Allows plugins to get access to APIs available in start inside async handlers, such as [App.mount](./kibana-plugin-core-public.app.mount.md). Promise will not resolve until Core and plugin dependencies have completed start. | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md deleted file mode 100644 index 2626086355a98c..00000000000000 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsaggs.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsAggs](./kibana-plugin-core-public.savedobjectsaggs.md) - -## SavedObjectsAggs type - -Signature: - -```typescript -export declare type SavedObjectsAggs = rt.TypeOf; -``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md deleted file mode 100644 index f09d7922e28b97..00000000000000 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.aggs.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindOptions](./kibana-plugin-core-public.savedobjectsfindoptions.md) > [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) - -## SavedObjectsFindOptions.aggs property - -Signature: - -```typescript -aggs?: SavedObjectsAggs; -``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md index 80b3b2b5cbbfc9..8bd87c2f6ea35f 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md @@ -15,7 +15,6 @@ export interface SavedObjectsFindOptions | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-core-public.savedobjectsfindoptions.aggs.md) | SavedObjectsAggs | | | [defaultSearchOperator](./kibana-plugin-core-public.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | The search operator to use with the provided filter. Defaults to OR | | [fields](./kibana-plugin-core-public.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-public.savedobjectsfindoptions.filter.md) | string | KueryNode | | diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 178393baad8c62..adbb2460dc80aa 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -285,7 +285,6 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | Type definition for a Saved Object attribute value | | [SavedObjectAttributeSingle](./kibana-plugin-core-server.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | | [SavedObjectMigrationFn](./kibana-plugin-core-server.savedobjectmigrationfn.md) | A migration function for a [saved object type](./kibana-plugin-core-server.savedobjectstype.md) used to migrate it to a given version | -| [SavedObjectsAggs](./kibana-plugin-core-server.savedobjectsaggs.md) | | | [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) | Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a references root property. | | [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.\#\# SavedObjectsClient errorsSince the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the isXYZError() helpers exposed at SavedObjectsErrorHelpers should be used to understand and manage error responses from the SavedObjectsClient.Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for error.body.error.type or doing substring checks on error.body.error.reason, just use the helpers to understand the meaning of the error:\`\`\`js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }// always rethrow the error unless you handle it throw error; \`\`\`\#\#\# 404s from missing indexFrom the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) | | [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md deleted file mode 100644 index 84f9e138c5408c..00000000000000 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsaggs.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsAggs](./kibana-plugin-core-server.savedobjectsaggs.md) - -## SavedObjectsAggs type - -Signature: - -```typescript -export declare type SavedObjectsAggs = rt.TypeOf; -``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md deleted file mode 100644 index 25588f623656a1..00000000000000 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.aggs.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) > [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) - -## SavedObjectsFindOptions.aggs property - -Signature: - -```typescript -aggs?: SavedObjectsAggs; -``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index 3587d57069124e..d393d579dbdd24 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -15,7 +15,6 @@ export interface SavedObjectsFindOptions | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-core-server.savedobjectsfindoptions.aggs.md) | SavedObjectsAggs | | | [defaultSearchOperator](./kibana-plugin-core-server.savedobjectsfindoptions.defaultsearchoperator.md) | 'AND' | 'OR' | The search operator to use with the provided filter. Defaults to OR | | [fields](./kibana-plugin-core-server.savedobjectsfindoptions.fields.md) | string[] | An array of fields to include in the results | | [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | string | KueryNode | | diff --git a/src/core/public/index.ts b/src/core/public/index.ts index 8fb838f4d5f174..564bbd712c535d 100644 --- a/src/core/public/index.ts +++ b/src/core/public/index.ts @@ -115,7 +115,6 @@ export { } from './application'; export { - SavedObjectsAggs, SavedObjectsBatchResponse, SavedObjectsBulkCreateObject, SavedObjectsBulkCreateOptions, diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 6defd281180f87..b7cbd22c870b05 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -31,7 +31,6 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { PublicUiSettingsParams as PublicUiSettingsParams_2 } from 'src/core/server/types'; import React from 'react'; import { RecursiveReadonly } from '@kbn/utility-types'; -import * as rt from 'io-ts'; import * as Rx from 'rxjs'; import { ShallowPromise } from '@kbn/utility-types'; import { TransportRequestOptions } from '@elastic/elasticsearch/lib/Transport'; @@ -1030,12 +1029,6 @@ export interface SavedObjectReference { type: string; } -// Warning: (ae-forgotten-export) The symbol "SavedObjectsAggsRt" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "SavedObjectsAggs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type SavedObjectsAggs = rt.TypeOf; - // @public (undocumented) export interface SavedObjectsBaseOptions { namespace?: string; @@ -1116,8 +1109,8 @@ export interface SavedObjectsCreateOptions { // @public (undocumented) export interface SavedObjectsFindOptions { - // (undocumented) - aggs?: SavedObjectsAggs; + // @alpha + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts diff --git a/src/core/public/saved_objects/index.ts b/src/core/public/saved_objects/index.ts index 06f66fdad24839..cc8fce0884ddf9 100644 --- a/src/core/public/saved_objects/index.ts +++ b/src/core/public/saved_objects/index.ts @@ -18,7 +18,6 @@ */ export { - SavedObjectsAggs, SavedObjectsBatchResponse, SavedObjectsBulkCreateObject, SavedObjectsBulkCreateOptions, diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 98b228150012e1..777a2647d9e85d 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -24,7 +24,6 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { SavedObject, SavedObjectReference, - SavedObjectsAggs, SavedObjectsClientContract as SavedObjectsApi, SavedObjectsFindOptions as SavedObjectFindOptionsServer, SavedObjectsMigrationVersion, @@ -40,9 +39,6 @@ type SavedObjectsFindOptions = Omit< type PromiseType> = T extends Promise ? U : never; -/** @public */ -export { SavedObjectsAggs }; - export interface SavedObjectsCreateOptions { /** * (Not recommended) Specify an id instead of having the saved objects service generate one for you. @@ -355,6 +351,12 @@ export class SavedObjectsClient { query.has_reference = JSON.stringify(query.has_reference); } + // `aggs` is a structured object. we need to stringify it before sending it, as `fetch` + // is not doing it implicitly. + if (query.aggs) { + query.aggs = JSON.stringify(query.aggs); + } + const request: ReturnType = this.savedObjectsFetch(path, { method: 'GET', query, diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 9d780c1de86224..7b19c3a6867579 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -241,7 +241,6 @@ export { } from './plugins'; export { - SavedObjectsAggs, SavedObjectsBulkCreateObject, SavedObjectsBulkGetObject, SavedObjectsBulkUpdateObject, diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index d703f0103d4116..1b2e7a445548a7 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -75,7 +75,7 @@ export const registerFindRoute = (router: IRouter) => { hasReferenceOperator: query.has_reference_operator, fields: typeof query.fields === 'string' ? [query.fields] : query.fields, filter: query.filter, - aggs: query.aggs != null ? JSON.parse(query.aggs) : undefined, + aggs: query.aggs ? JSON.parse(query.aggs) : undefined, namespaces, }); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index 3b9bb6332e8b9c..a273772367a382 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -19,7 +19,6 @@ import { validateGetSavedObjectsAggs } from './aggs_utils'; import { mockMappings } from './filter_utils.test'; -import { SavedObjectsAggs } from './saved_objects_aggs_types'; describe('Filter Utils', () => { describe('#validateGetSavedObjectsAggs', () => { @@ -38,7 +37,8 @@ describe('Filter Utils', () => { }, }); }); - test('Validate a nested simple aggregations', () => { + + test('Validate a nested field in simple aggregations', () => { expect( validateGetSavedObjectsAggs( ['alert'], @@ -54,6 +54,72 @@ describe('Filter Utils', () => { }); }); + test('Validate a nested aggregations', () => { + expect( + validateGetSavedObjectsAggs( + ['alert'], + { + aggName: { + cardinality: { + field: 'alert.attributes.actions.group', + aggs: { + aggName: { + max: { field: 'alert.attributes.actions.group' }, + }, + }, + }, + }, + }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', + aggs: { + aggName: { + max: { + field: 'alert.actions.group', + }, + }, + }, + }, + }, + }); + }); + + test('Validate an aggregation without the attribute field', () => { + expect( + validateGetSavedObjectsAggs( + ['alert'], + { aggName: { terms: { 'alert.attributes.actions.group': ['myFriend', 'snoopy'] } } }, + mockMappings + ) + ).toEqual({ + aggName: { + terms: { + 'alert.actions.group': ['myFriend', 'snoopy'], + }, + }, + }); + }); + + test('Validate a filter term aggregations', () => { + expect( + validateGetSavedObjectsAggs( + ['foo'], + { aggName: { filter: { term: { 'foo.attributes.bytes': 10 } } } }, + mockMappings + ) + ).toEqual({ + aggName: { + filter: { + term: { 'foo.attributes.bytes': 10 }, + }, + }, + }); + }); + test('Throw an error when types is not allowed', () => { expect(() => { validateGetSavedObjectsAggs( @@ -68,7 +134,37 @@ describe('Filter Utils', () => { }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed: Bad Request"`); }); - test('Throw an error when aggregation is not defined in SavedObjectsAggs', () => { + test('Throw an error when add an invalid attributes ', () => { + expect(() => { + validateGetSavedObjectsAggs( + ['foo'], + { + aggName: { + max: { field: 'foo.attributes.bytes', notValid: 'yesIamNotValid' }, + }, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot( + `"notValid attribute is not supported in max saved objects aggregation: Bad Request"` + ); + }); + + test('Throw an error when an attributes is not defined correctly', () => { + expect(() => + validateGetSavedObjectsAggs( + ['alert'], + { + aggName: { + terms: { 'alert.attributes.actions.group': ['myFriend', 'snoopy'], missing: 0 }, + }, + }, + mockMappings + ) + ).toThrowErrorMatchingInlineSnapshot(`"Invalid value 0 supplied to : string: Bad Request"`); + }); + + test('Throw an error when aggregation is not defined in SavedObjectsAggs', () => { expect(() => { validateGetSavedObjectsAggs( ['foo'], @@ -76,15 +172,38 @@ describe('Filter Utils', () => { aggName: { MySuperAgg: { field: 'foo.attributes.bytes' }, }, - } as SavedObjectsAggs, + }, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid value {\\"aggName\\":{\\"MySuperAgg\\":{\\"field\\":\\"foo.attributes.bytes\\"}}}, excess properties: [\\"MySuperAgg\\"]: Bad Request"` + `"This aggregation MySuperAgg is not valid or we did not defined it yet: Bad Request"` ); }); - test('Throw an error when you add attributes who are not defined in SavedObjectsAggs', () => { + test('Throw an error when children aggregation is not defined in SavedObjectsAggs', () => { + expect(() => { + validateGetSavedObjectsAggs( + ['foo'], + { + aggName: { + cardinality: { + field: 'foo.attributes.bytes', + aggs: { + aggName: { + MySuperAgg: { field: 'alert.attributes.actions.group' }, + }, + }, + }, + }, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot( + `"This aggregation MySuperAgg is not valid or we did not defined it yet: Bad Request"` + ); + }); + + test('Throw an error when you add the script attribute who are not defined in SavedObjectsAggs', () => { expect(() => { validateGetSavedObjectsAggs( ['alert'], @@ -93,11 +212,37 @@ describe('Filter Utils', () => { cardinality: { field: 'alert.attributes.actions.group' }, script: 'I want to access that I should not', }, - } as SavedObjectsAggs, + }, + mockMappings + ); + }).toThrowErrorMatchingInlineSnapshot( + `"script attribute is not supported in saved objects aggregation: Bad Request"` + ); + }); + + test('Throw an error when you add the script attribute in a nested aggregations who are not defined in SavedObjectsAggs', () => { + expect(() => { + validateGetSavedObjectsAggs( + ['alert'], + { + aggName: { + cardinality: { + field: 'alert.attributes.actions.group', + aggs: { + aggName: { + max: { + field: 'alert.attributes.actions.group', + script: 'I want to access that I should not', + }, + }, + }, + }, + }, + }, mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid value {\\"aggName\\":{\\"cardinality\\":{\\"field\\":\\"alert.attributes.actions.group\\"},\\"script\\":\\"I want to access that I should not\\"}}, excess properties: [\\"script\\"]: Bad Request"` + `"script attribute is not supported in saved objects aggregation: Bad Request"` ); }); }); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index b88ce5b50dc8bd..2642da56f6b826 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -21,38 +21,82 @@ import { IndexMapping } from '../../mappings'; import { SavedObjectsErrorHelpers } from './errors'; import { hasFilterKeyError } from './filter_utils'; -import { SavedObjectsAggs, validateSavedObjectTypeAggs } from './saved_objects_aggs_types'; +import { savedObjectsAggs, validateSavedObjectsTypeAggs } from './saved_objects_aggs_types'; export const validateGetSavedObjectsAggs = ( allowedTypes: string[], - aggs: SavedObjectsAggs, + aggs: Record, indexMapping: IndexMapping -): SavedObjectsAggs => { - validateSavedObjectTypeAggs(aggs); +) => { return validateGetAggFieldValue(allowedTypes, aggs, indexMapping); }; const validateGetAggFieldValue = ( allowedTypes: string[], aggs: any, - indexMapping: IndexMapping -): SavedObjectsAggs => { + indexMapping: IndexMapping, + lastKey?: string, + aggType?: string +): unknown => { return Object.keys(aggs).reduce((acc, key) => { - if (key === 'field') { - const error = hasFilterKeyError(aggs[key], allowedTypes, indexMapping); + if (key === 'script') { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'script attribute is not supported in saved objects aggregation' + ); + } + if (typeof aggs[key] === 'object' && aggType === undefined && savedObjectsAggs[key]) { + return { + ...acc, + [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping, key, key), + }; + } else if ( + typeof aggs[key] === 'object' && + (['aggs', 'aggregations'].includes(key) || aggType === undefined) + ) { + return { + ...acc, + [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping, key, undefined), + }; + } else if ( + key !== 'field' && + aggType && + savedObjectsAggs[aggType] !== undefined && + savedObjectsAggs[aggType][key] !== undefined + ) { + validateSavedObjectsTypeAggs(savedObjectsAggs[aggType][key], aggs[key]); + return { + ...acc, + [key]: aggs[key], + }; + } else { + if (aggType === undefined || savedObjectsAggs[aggType] === undefined) { + throw SavedObjectsErrorHelpers.createBadRequestError( + `This aggregation ${lastKey} is not valid or we did not defined it yet` + ); + } + const error = hasFilterKeyError( + key === 'field' ? aggs[key] : key, + allowedTypes, + indexMapping + ); if (error != null) { + if ( + aggType !== undefined && + savedObjectsAggs[aggType] !== undefined && + savedObjectsAggs[aggType][key] === undefined + ) { + throw SavedObjectsErrorHelpers.createBadRequestError( + `${key} attribute is not supported in ${aggType} saved objects aggregation` + ); + } throw SavedObjectsErrorHelpers.createBadRequestError(error); } return { ...acc, - [key]: aggs[key].replace('.attributes', ''), + ...(key === 'field' + ? { [key]: aggs[key].replace('.attributes', '') } + : { [key.replace('.attributes', '')]: aggs[key] }), }; - } else if (typeof aggs[key] === 'object') { - return { ...acc, [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping) }; } - return { - ...acc, - [key]: aggs[key], - }; }, {}); }; diff --git a/src/core/server/saved_objects/service/lib/index.ts b/src/core/server/saved_objects/service/lib/index.ts index 8972c57b2464aa..eae8c5ef2e10cf 100644 --- a/src/core/server/saved_objects/service/lib/index.ts +++ b/src/core/server/saved_objects/service/lib/index.ts @@ -31,5 +31,4 @@ export { export { SavedObjectsErrorHelpers } from './errors'; -export { SavedObjectsAggs } from './saved_objects_aggs_types'; export { SavedObjectsUtils } from './utils'; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 830b9cdd1136ae..8fcd4cbc4f3cf6 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -765,26 +765,25 @@ export class SavedObjectsRepository { } let kueryNode; - - try { - if (filter) { + if (filter) { + try { kueryNode = validateConvertFilterToKueryNode(allowedTypes, filter, this._mappings); - } - } catch (e) { - if (e.name === 'KQLSyntaxError') { - throw SavedObjectsErrorHelpers.createBadRequestError('KQLSyntaxError: ' + e.message); - } else { - throw e; + } catch (e) { + if (e.name === 'KQLSyntaxError') { + throw SavedObjectsErrorHelpers.createBadRequestError('KQLSyntaxError: ' + e.message); + } else { + throw e; + } } } let aggsObject = null; - try { - if (aggs) { + if (aggs) { + try { aggsObject = validateGetSavedObjectsAggs(allowedTypes, aggs, this._mappings); + } catch (e) { + throw e; } - } catch (e) { - throw e; } const esOptions = { diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts index 8be958a94b4c1e..cb9c5febce6e32 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts @@ -19,30 +19,31 @@ import * as rt from 'io-ts'; -import { FieldBasicRT } from '../helpers'; +import { fieldBasic } from '../helpers'; -export const BucketAggsTypeRt = rt.partial({ - filter: rt.type({ - term: rt.record(rt.string, rt.string), - }), - histogram: rt.intersection([ - FieldBasicRT, - rt.type({ interval: rt.number }), - rt.partial({ - min_doc_count: rt.number, - extended_bounds: rt.type({ min: rt.number, max: rt.number }), - keyed: rt.boolean, - missing: rt.number, - order: rt.record(rt.string, rt.literal('asc', 'desc')), - }), - ]), - terms: rt.intersection([ - FieldBasicRT, - rt.partial({ - field: rt.string, - size: rt.number, - show_term_doc_count_error: rt.boolean, - order: rt.record(rt.string, rt.literal('asc', 'desc')), - }), - ]), -}); +export const bucketAggsType: Record> = { + filter: { + term: rt.record(rt.string, rt.any), + }, + histogram: { + ...fieldBasic, + interval: rt.number, + min_doc_count: rt.number, + extended_bounds: rt.type({ min: rt.number, max: rt.number }), + keyed: rt.boolean, + missing: rt.number, + order: rt.record(rt.string, rt.literal('asc', 'desc')), + }, + terms: { + ...fieldBasic, + collect_mode: rt.string, + exclude: rt.unknown, + execution_hint: rt.string, + include: rt.unknown, + missing: rt.string, + min_doc_count: rt.number, + size: rt.number, + show_term_doc_count_error: rt.boolean, + order: rt.record(rt.string, rt.literal('asc', 'desc')), + }, +}; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts deleted file mode 100644 index 2feb7fa6c7f7e3..00000000000000 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import * as rt from 'io-ts'; -import { PathReporter } from 'io-ts/lib/PathReporter'; - -import { excess } from './helpers'; - -const runDecode = (codec: rt.Type, data: any) => { - const result = codec.decode(data); - return PathReporter.report(result); -}; - -describe('Saved object aggs helpers', () => { - describe('happy path', () => { - test('excess Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.partial({ - max: rt.type({ field: rt.string }), - }) - ) - ); - - expect(runDecode(codec, { aggName: { max: { field: 'hi' } } })).toEqual(['No errors!']); - }); - - test('excess Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.intersection([ - rt.partial({ - max: rt.type({ field: rt.string }), - }), - rt.partial({ - min: rt.type({ field: rt.string }), - }), - ]) - ) - ); - - expect(runDecode(codec, { aggName: { min: { field: 'hi' } } })).toEqual(['No errors!']); - }); - - test('When you intersection as a DictionnaryType', () => { - const codec = excess( - rt.record( - rt.string, - rt.intersection([ - rt.partial({ - max: rt.type({ field: rt.string }), - }), - rt.partial({ - filter: rt.type({ field: rt.string }), - aggs: rt.record( - rt.string, - rt.partial({ - min: rt.type({ field: rt.string }), - }) - ), - }), - ]) - ) - ); - - expect( - runDecode(codec, { - aggName: { filter: { field: 'hi' }, aggs: { aggNewName: { min: { field: 'low' } } } }, - }) - ).toEqual(['No errors!']); - }); - }); - - describe('Errors', () => { - test('throw error when you add an attributes who is not expected for Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.partial({ - max: rt.type({ field: rt.string }), - }) - ) - ); - - expect(runDecode(codec, { aggName: { max: { field: 'hi', script: '' } } })).toEqual([ - 'Invalid value {"aggName":{"max":{"field":"hi","script":""}}}, excess properties: ["script"]', - ]); - }); - - test('throw error when you add an attributes who is not expected for Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.intersection([ - rt.partial({ - max: rt.type({ field: rt.string }), - }), - rt.partial({ - min: rt.type({ field: rt.string }), - }), - ]) - ) - ); - - expect(runDecode(codec, { aggName: { min: { field: 'hi', script: 'field' } } })).toEqual([ - 'Invalid value {"aggName":{"min":{"field":"hi","script":"field"}}}, excess properties: ["script"]', - ]); - }); - - test('throw error when you do not match types for Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.partial({ - max: rt.type({ field: rt.string }), - }) - ) - ); - - expect(runDecode(codec, { aggName: { max: { field: 33 } } })).toEqual([ - 'Invalid value 33 supplied to : { [K in string]: Partial<{ max: { field: string } }> }/aggName: Partial<{ max: { field: string } }>/max: { field: string }/field: string', - ]); - }); - - test('throw error when when you do not match types for Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.intersection([ - rt.partial({ - max: rt.type({ field: rt.string }), - }), - rt.partial({ - min: rt.type({ field: rt.string }), - }), - ]) - ) - ); - - expect(runDecode(codec, { aggName: { min: { field: 33 } } })).toEqual([ - 'Invalid value 33 supplied to : { [K in string]: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>) }/aggName: (Partial<{ max: { field: string } }> & Partial<{ min: { field: string } }>)/1: Partial<{ min: { field: string } }>/min: { field: string }/field: string', - ]); - }); - - test('throw error when you add an attributes in your second agg who is not expected for Record', () => { - const codec = excess( - rt.record( - rt.string, - rt.intersection([ - rt.partial({ - max: rt.type({ field: rt.string }), - }), - rt.partial({ - filter: rt.type({ field: rt.string }), - aggs: rt.record( - rt.string, - rt.partial({ - min: rt.type({ field: rt.string }), - }) - ), - }), - ]) - ) - ); - - expect( - runDecode(codec, { - aggName: { - filter: { field: 'hi' }, - aggs: { aggNewName: { min: { field: 'low' }, script: 'error' } }, - }, - }) - ).toEqual([ - 'Invalid value {"aggName":{"filter":{"field":"hi"},"aggs":{"aggNewName":{"min":{"field":"low"},"script":"error"}}}}, excess properties: ["script"]', - ]); - }); - }); -}); diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts index 6475d97ec6e797..6485639636ec34 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts @@ -17,100 +17,14 @@ * under the License. */ -import { either } from 'fp-ts/lib/Either'; import * as rt from 'io-ts'; import { failure } from 'io-ts/lib/PathReporter'; -import { get } from 'lodash'; type ErrorFactory = (message: string) => Error; +export const fieldBasic = { field: rt.string }; export const FieldBasicRT = rt.type({ field: rt.string }); export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { throw createError(failure(errors).join('\n')); }; - -const getProps = (codec: rt.HasProps | rt.RecordC): rt.Props | null => { - if (codec == null) { - return null; - } - switch (codec._tag) { - case 'DictionaryType': - if (codec.codomain.props != null) { - return codec.codomain.props; - } - const types: rt.HasProps[] = codec.codomain.types; - return types.reduce((props, type) => Object.assign(props, getProps(type)), {}); - case 'RefinementType': - case 'ReadonlyType': - return getProps(codec.type); - case 'InterfaceType': - case 'StrictType': - case 'PartialType': - return codec.props; - case 'IntersectionType': - return codec.types.reduce( - (props, type) => Object.assign(props, getProps(type)), - {} - ); - default: - return null; - } -}; - -const getExcessProps = ( - props: rt.Props | rt.RecordC, - r: Record -): string[] => - Object.keys(r).reduce((acc, k) => { - const codecChildren = get(props, [k]); - const childrenProps = getProps(codecChildren); - const childrenObject = r[k] as Record; - if (codecChildren != null && childrenProps != null && codecChildren._tag === 'DictionaryType') { - const keys = Object.keys(childrenObject); - return [ - ...acc, - ...keys.reduce( - (kAcc, i) => [...kAcc, ...getExcessProps(childrenProps, childrenObject[i])], - [] - ), - ]; - } - if (props.hasOwnProperty(k) && childrenProps != null) { - return [...acc, ...getExcessProps(childrenProps, childrenObject)]; - } else if (!props.hasOwnProperty(k)) { - return [...acc, k]; - } - return acc; - }, []); - -export const excess = (codec: rt.RecordC): rt.InterfaceType => { - const codecProps = getProps(codec); - - const r = new rt.DictionaryType( - codec.name, - codec.is, - (u, c) => - either.chain(codec.validate(u, c), (o: Record) => { - if (codecProps == null) { - return rt.failure(u, c, `Invalid Aggs object ${JSON.stringify(u)}`); - } - const keys = Object.keys(o); - const ex = keys.reduce((acc, k) => { - return [...acc, ...getExcessProps(codecProps, o[k])]; - }, []); - - return ex.length > 0 - ? rt.failure( - u, - c, - `Invalid value ${JSON.stringify(u)}, excess properties: ${JSON.stringify(ex)}` - ) - : codec.validate(u, c); - }), - codec.encode, - codec.domain, - codec.codomain - ); - return r as any; -}; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts index afe8e3b8b4a18e..a60b38ed4db453 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts @@ -23,24 +23,20 @@ import { pipe } from 'fp-ts/lib/pipeable'; import * as rt from 'io-ts'; -import { BucketAggsTypeRt } from './bucket_aggs'; -import { MetricsAggsTypeRt } from './metrics_aggs'; +import { bucketAggsType } from './bucket_aggs'; +import { metricsAggsType } from './metrics_aggs'; import { SavedObjectsErrorHelpers } from '../errors'; -import { excess, throwErrors } from './helpers'; +import { throwErrors } from './helpers'; -const AllAggsRt = rt.intersection([BucketAggsTypeRt, MetricsAggsTypeRt]); - -const SavedObjectsAggsRt = rt.record( - rt.string, - rt.intersection([AllAggsRt, rt.partial({ aggs: AllAggsRt })]) -); - -export type SavedObjectsAggs = rt.TypeOf; +export const savedObjectsAggs = { + ...metricsAggsType, + ...bucketAggsType, +}; -export const validateSavedObjectTypeAggs = (aggObjects: SavedObjectsAggs) => { +export const validateSavedObjectsTypeAggs = (rtType: rt.Any, aggObject: unknown) => { pipe( - excess(SavedObjectsAggsRt).decode(aggObjects), + rtType.decode(aggObject), fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) ); }; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts index 169d8d5c426a0f..7de8ed12dcc69a 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts @@ -19,7 +19,7 @@ import * as rt from 'io-ts'; -import { FieldBasicRT } from '../helpers'; +import { fieldBasic, FieldBasicRT } from '../helpers'; /* * Types for Metrics Aggregations @@ -39,24 +39,24 @@ import { FieldBasicRT } from '../helpers'; * - Median Absolute Deviation Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-median-absolute-deviation-aggregation.html */ -const MinMaxAggBodyRt = rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]); - -export const MetricsAggsTypeRt = rt.partial({ - avg: FieldBasicRT, - weighted_avg: rt.intersection([ - rt.type({ - value: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), - weight: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), - }), - rt.partial({ - format: rt.string, - value_type: rt.string, - }), - ]), - cardinality: FieldBasicRT, - max: MinMaxAggBodyRt, - min: MinMaxAggBodyRt, - top_hits: rt.partial({ +export const metricsAggsType: Record> = { + avg: fieldBasic, + weighted_avg: { + value: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), + weight: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), + format: rt.string, + value_type: rt.string, + }, + cardinality: fieldBasic, + max: { + ...fieldBasic, + missing: rt.number, + }, + min: { + ...fieldBasic, + missing: rt.number, + }, + top_hits: { explain: rt.boolean, from: rt.string, highlight: rt.any, @@ -70,5 +70,5 @@ export const MetricsAggsTypeRt = rt.partial({ includes: rt.array(rt.string), excludes: rt.array(rt.string), }), - }), -}); + }, +}; diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index d6ef6a29a7b066..601ac8f8911584 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -17,7 +17,7 @@ * under the License. */ -import { ISavedObjectsRepository, SavedObjectsAggs } from './lib'; +import { ISavedObjectsRepository } from './lib'; import { SavedObject, SavedObjectError, @@ -285,12 +285,6 @@ export interface SavedObjectsUpdateResponse references: SavedObjectReference[] | undefined; } -/** - * - * @public - */ -export { SavedObjectsAggs }; - /** * * @public diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index a2cf8c5a593ee1..a9b0fd5ee9e87b 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -17,7 +17,7 @@ * under the License. */ -import { SavedObjectsClient, SavedObjectsAggs } from './service/saved_objects_client'; +import { SavedObjectsClient } from './service/saved_objects_client'; import { SavedObjectsTypeMappingDefinition } from './mappings'; import { SavedObjectMigrationMap } from './migrations'; @@ -116,7 +116,7 @@ export interface SavedObjectsFindOptions { * aggs = { type_count: { max: { field: 'dashboard.attributes.version' } } }; * SavedObjects.find({type: 'dashboard', aggs: '%7B%22type_count%22%3A%7B%22max%22%3A%7B%22field%22%3A%22dashboard.attributes.version%22%7D%7D%7D'}) */ - aggs?: SavedObjectsAggs; + aggs?: Record; namespaces?: string[]; /** * This map defines each type to search for, and the namespace(s) to search for the type in; this is only intended to be used by a saved diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 0c2984c54c34c0..fbfe177c65a9d6 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -132,7 +132,6 @@ import { RenderSearchTemplateParams } from 'elasticsearch'; import { Request } from '@hapi/hapi'; import { ResponseObject } from '@hapi/hapi'; import { ResponseToolkit } from '@hapi/hapi'; -import * as rt from 'io-ts'; import { SchemaTypeError } from '@kbn/config-schema'; import { ScrollParams } from 'elasticsearch'; import { SearchParams } from 'elasticsearch'; @@ -1893,12 +1892,6 @@ export interface SavedObjectsAddToNamespacesResponse { namespaces: string[]; } -// Warning: (ae-forgotten-export) The symbol "SavedObjectsAggsRt" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "SavedObjectsAggs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type SavedObjectsAggs = rt.TypeOf; - // Warning: (ae-forgotten-export) The symbol "SavedObjectDoc" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "Referencable" needs to be exported by the entry point index.d.ts // @@ -2195,8 +2188,8 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions { - // (undocumented) - aggs?: SavedObjectsAggs; + // @alpha + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 46a45dc10665ec..fc9b8d4839ea3c 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -69,7 +69,6 @@ import { Reporter } from '@kbn/analytics'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestStatistics as RequestStatistics_2 } from 'src/plugins/inspector/common'; import { Required } from '@kbn/utility-types'; -import * as rt from 'io-ts'; import * as Rx from 'rxjs'; import { SavedObject } from 'kibana/server'; import { SavedObject as SavedObject_2 } from 'src/core/server'; diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md index 03838b870d4fba..f3f3682404e326 100644 --- a/src/plugins/embeddable/public/public.api.md +++ b/src/plugins/embeddable/public/public.api.md @@ -62,7 +62,6 @@ import React from 'react'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter as RequestAdapter_2 } from 'src/plugins/inspector/common'; import { Required } from '@kbn/utility-types'; -import * as rt from 'io-ts'; import * as Rx from 'rxjs'; import { SavedObject as SavedObject_2 } from 'kibana/server'; import { SavedObject as SavedObject_3 } from 'src/core/server'; diff --git a/test/api_integration/apis/index.js b/test/api_integration/apis/index.js index d07c099634005d..65a388d5a52e8b 100644 --- a/test/api_integration/apis/index.js +++ b/test/api_integration/apis/index.js @@ -19,20 +19,20 @@ export default function ({ loadTestFile }) { describe('apis', () => { - loadTestFile(require.resolve('./core')); - loadTestFile(require.resolve('./general')); - loadTestFile(require.resolve('./home')); - loadTestFile(require.resolve('./index_patterns')); - loadTestFile(require.resolve('./kql_telemetry')); - loadTestFile(require.resolve('./saved_objects_management')); + // loadTestFile(require.resolve('./core')); + // loadTestFile(require.resolve('./general')); + // loadTestFile(require.resolve('./home')); + // loadTestFile(require.resolve('./index_patterns')); + // loadTestFile(require.resolve('./kql_telemetry')); + // loadTestFile(require.resolve('./saved_objects_management')); loadTestFile(require.resolve('./saved_objects')); - loadTestFile(require.resolve('./scripts')); - loadTestFile(require.resolve('./search')); - loadTestFile(require.resolve('./shorten')); - loadTestFile(require.resolve('./suggestions')); - loadTestFile(require.resolve('./status')); - loadTestFile(require.resolve('./stats')); - loadTestFile(require.resolve('./ui_metric')); - loadTestFile(require.resolve('./telemetry')); + // loadTestFile(require.resolve('./scripts')); + // loadTestFile(require.resolve('./search')); + // loadTestFile(require.resolve('./shorten')); + // loadTestFile(require.resolve('./suggestions')); + // loadTestFile(require.resolve('./status')); + // loadTestFile(require.resolve('./stats')); + // loadTestFile(require.resolve('./ui_metric')); + // loadTestFile(require.resolve('./telemetry')); }); } diff --git a/test/api_integration/apis/saved_objects/find.js b/test/api_integration/apis/saved_objects/find.js index 0ea95448f87b9b..7e0ca6d84e33d0 100644 --- a/test/api_integration/apis/saved_objects/find.js +++ b/test/api_integration/apis/saved_objects/find.js @@ -346,7 +346,7 @@ export default function ({ getService }) { JSON.stringify({ type_count: { max: { - field: 'dashboard.attributes.version', + field: 'visualization.attributes.version', script: 'Oh yes I am going to a script', }, }, @@ -359,7 +359,7 @@ export default function ({ getService }) { expect(resp.body).to.eql({ error: 'Bad Request', message: - 'Invalid value {"type_count":{"max":{"field":"dashboard.attributes.version","script":"Oh yes I am going to a script"}}}, excess properties: ["script"]: Bad Request', + 'script attribute is not supported in saved objects aggregation: Bad Request', statusCode: 400, }); })); diff --git a/test/api_integration/apis/saved_objects/index.js b/test/api_integration/apis/saved_objects/index.js index ad6c3749181dd5..bcdaa80b6334ec 100644 --- a/test/api_integration/apis/saved_objects/index.js +++ b/test/api_integration/apis/saved_objects/index.js @@ -19,17 +19,17 @@ export default function ({ loadTestFile }) { describe('saved_objects', () => { - loadTestFile(require.resolve('./bulk_create')); - loadTestFile(require.resolve('./bulk_get')); - loadTestFile(require.resolve('./create')); - loadTestFile(require.resolve('./delete')); - loadTestFile(require.resolve('./export')); + // loadTestFile(require.resolve('./bulk_create')); + // loadTestFile(require.resolve('./bulk_get')); + // loadTestFile(require.resolve('./create')); + // loadTestFile(require.resolve('./delete')); + // loadTestFile(require.resolve('./export')); loadTestFile(require.resolve('./find')); - loadTestFile(require.resolve('./get')); - loadTestFile(require.resolve('./import')); - loadTestFile(require.resolve('./resolve_import_errors')); - loadTestFile(require.resolve('./update')); - loadTestFile(require.resolve('./bulk_update')); - loadTestFile(require.resolve('./migrations')); + // loadTestFile(require.resolve('./get')); + // loadTestFile(require.resolve('./import')); + // loadTestFile(require.resolve('./resolve_import_errors')); + // loadTestFile(require.resolve('./update')); + // loadTestFile(require.resolve('./bulk_update')); + // loadTestFile(require.resolve('./migrations')); }); } From 4e319b3886c5c52354ed48b17ef089298d840d2d Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 23 Nov 2020 10:32:12 -0500 Subject: [PATCH 19/52] Validate multiple items nested filter query through KueryNode --- .../service/lib/filter_utils.test.ts | 28 +++++++++++++++++++ .../saved_objects/service/lib/filter_utils.ts | 12 +++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index 3e4845f526df7c..bad5a10acffa01 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -494,5 +494,33 @@ describe('Filter Utils', () => { }, ]); }); + + test('Validate multiple items nested filter query through KueryNode', () => { + const validationObject = validateFilterKueryNode({ + astFilter: esKuery.fromKueryExpression( + 'alert.attributes.actions:{ actionTypeId: ".server-log" AND actionRef: "foo" }' + ), + types: ['alert'], + indexMapping: mockMappings, + }); + + // nodes will have errors in the array + expect(validationObject).toEqual([ + { + astPath: 'arguments.1.arguments.0', + error: null, + isSavedObjectAttr: false, + key: 'alert.attributes.actions.actionTypeId', + type: 'alert', + }, + { + astPath: 'arguments.1.arguments.1', + error: null, + isSavedObjectAttr: false, + key: 'alert.attributes.actions.actionRef', + type: 'alert', + }, + ]); + }); }); }); diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index be36807f0d02b8..6e46fc3eff008e 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -19,6 +19,7 @@ import { set } from '@elastic/safer-lodash-set'; import { get } from 'lodash'; +import { string } from 'joi'; import { SavedObjectsErrorHelpers } from './errors'; import { IndexMapping } from '../../mappings'; // @ts-expect-error no ts @@ -120,7 +121,15 @@ export const validateFilterKueryNode = ({ return astFilter.arguments.reduce((kueryNode: string[], ast: KueryNode, index: number) => { if (hasNestedKey && ast.type === 'literal' && ast.value != null) { localNestedKeys = ast.value; + } else if (ast.type === 'literal' && ast.value && typeof ast.value === 'string') { + const key = ast.value.replace('.attributes', ''); + const mappingKey = 'properties.' + key.split('.').join('.properties.'); + const field = get(indexMapping, mappingKey); + if (field != null && field.type === 'nested') { + localNestedKeys = ast.value; + } } + if (ast.arguments) { const myPath = `${path}.${index}`; return [ @@ -132,7 +141,7 @@ export const validateFilterKueryNode = ({ storeValue: ast.type === 'function' && astFunctionType.includes(ast.function), path: `${myPath}.arguments`, hasNestedKey: ast.type === 'function' && ast.function === 'nested', - nestedKeys: localNestedKeys, + nestedKeys: localNestedKeys || nestedKeys, }), ]; } @@ -186,6 +195,7 @@ export const hasFilterKeyError = ( types: string[], indexMapping: IndexMapping ): string | null => { + // console.log('hasFilterKeyError', indexMapping, key) if (key == null) { return `The key is empty and needs to be wrapped by a saved object type like ${types.join()}`; } From 16de8751ab3143e76de7c6ea10bcd6f855263037 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 23 Nov 2020 10:34:27 -0500 Subject: [PATCH 20/52] remove unused import --- src/core/server/saved_objects/service/lib/filter_utils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index 6e46fc3eff008e..5c7e9dae70d7f0 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -19,7 +19,6 @@ import { set } from '@elastic/safer-lodash-set'; import { get } from 'lodash'; -import { string } from 'joi'; import { SavedObjectsErrorHelpers } from './errors'; import { IndexMapping } from '../../mappings'; // @ts-expect-error no ts @@ -195,7 +194,6 @@ export const hasFilterKeyError = ( types: string[], indexMapping: IndexMapping ): string | null => { - // console.log('hasFilterKeyError', indexMapping, key) if (key == null) { return `The key is empty and needs to be wrapped by a saved object type like ${types.join()}`; } From cb7ada373acb516b2bd29cf8a01850d491b35678 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Mon, 23 Nov 2020 10:52:02 -0500 Subject: [PATCH 21/52] review + put back test --- docs/api/saved-objects/find.asciidoc | 4 +- .../service/lib/filter_utils.test.ts | 42 +++++++++---------- .../saved_objects/service/lib/repository.ts | 1 - src/core/server/saved_objects/types.ts | 4 +- test/api_integration/apis/index.js | 28 ++++++------- .../apis/saved_objects/index.js | 22 +++++----- 6 files changed, 50 insertions(+), 51 deletions(-) diff --git a/docs/api/saved-objects/find.asciidoc b/docs/api/saved-objects/find.asciidoc index 03e2b3414a8df7..73a8fbf0a820f7 100644 --- a/docs/api/saved-objects/find.asciidoc +++ b/docs/api/saved-objects/find.asciidoc @@ -54,12 +54,12 @@ experimental[] Retrieve a paginated set of {kib} saved objects by various condit `filter`:: (Optional, string) The filter is a KQL string with the caveat that if you filter with an attribute from your type saved object. - It should look like that savedObjectType.attributes.title: "myTitle". However, If you used a direct attribute of a saved object like `updatedAt`, + It should look like that savedObjectType.attributes.title: "myTitle". However, If you used a direct attribute of a saved object like `updated_at`, you will have to define your filter like that savedObjectType.updatedAt > 2018-12-22. `aggs`:: (Optional, string) The aggs will support aggregation string with the caveat that your field from the aggregation will have the attribute from your type saved object, - it should look like this: savedObjectType.attributes.field. However, If you use a direct attribute of a saved object like updatedAt, you will have to define your filter like this: savedObjectType.updatedAt. + it should look like this: savedObjectType.attributes.field. However, If you use a direct attribute of a saved object like updatedAt, you will have to define your filter like this: savedObjectType.updated_at. NOTE: As objects change in {kib}, the results on each page of the response also change. Use the find API for traditional paginated results, but avoid using it to export large amounts of data. diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index bad5a10acffa01..df7cbda3210f66 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -23,7 +23,7 @@ import { validateFilterKueryNode, validateConvertFilterToKueryNode } from './fil export const mockMappings = { properties: { - updatedAt: { + updated_at: { type: 'date', }, foo: { @@ -104,12 +104,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo'], - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '(type: foo and updatedAt: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' + '(type: foo and updated_at: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' ) ); }); @@ -118,12 +118,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo', 'bar'], - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '(type: foo and updatedAt: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' + '(type: foo and updated_at: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' ) ); }); @@ -132,12 +132,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo', 'bar'], - '(bar.updatedAt: 5678654567 OR foo.updatedAt: 5678654567) and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or bar.attributes.description :*)', + '(bar.updated_at: 5678654567 OR foo.updated_at: 5678654567) and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or bar.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '((type: bar and updatedAt: 5678654567) or (type: foo and updatedAt: 5678654567)) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or bar.description :*)' + '((type: bar and updated_at: 5678654567) or (type: foo and updated_at: 5678654567)) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or bar.description :*)' ) ); }); @@ -156,11 +156,11 @@ describe('Filter Utils', () => { expect(() => { validateConvertFilterToKueryNode( ['foo', 'bar'], - 'updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"This key 'updatedAt' need to be wrapped by a saved object type like foo,bar: Bad Request"` + `"This key 'updated_at' need to be wrapped by a saved object type like foo,bar: Bad Request"` ); }); @@ -175,7 +175,7 @@ describe('Filter Utils', () => { test('Validate filter query through KueryNode - happy path', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -186,7 +186,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: null, isSavedObjectAttr: true, - key: 'foo.updatedAt', + key: 'foo.updated_at', type: 'foo', }, { @@ -250,7 +250,7 @@ describe('Filter Utils', () => { test('Return Error if key is not wrapper by a saved object type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -259,9 +259,9 @@ describe('Filter Utils', () => { expect(validationObject).toEqual([ { astPath: 'arguments.0', - error: "This key 'updatedAt' need to be wrapped by a saved object type like foo", + error: "This key 'updated_at' need to be wrapped by a saved object type like foo", isSavedObjectAttr: true, - key: 'updatedAt', + key: 'updated_at', type: null, }, { @@ -305,7 +305,7 @@ describe('Filter Utils', () => { test('Return Error if key of a saved object type is not wrapped with attributes', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.description :*)' + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -316,7 +316,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: null, isSavedObjectAttr: true, - key: 'foo.updatedAt', + key: 'foo.updated_at', type: 'foo', }, { @@ -362,7 +362,7 @@ describe('Filter Utils', () => { test('Return Error if filter is not using an allowed type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'bar.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'bar.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -373,7 +373,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: 'This type bar is not allowed', isSavedObjectAttr: true, - key: 'bar.updatedAt', + key: 'bar.updated_at', type: 'bar', }, { @@ -417,7 +417,7 @@ describe('Filter Utils', () => { test('Return Error if filter is using an non-existing key in the index patterns of the saved object type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt33: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.header: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'foo.updated_at33: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.header: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -426,9 +426,9 @@ describe('Filter Utils', () => { expect(validationObject).toEqual([ { astPath: 'arguments.0', - error: "This key 'foo.updatedAt33' does NOT exist in foo saved object index patterns", + error: "This key 'foo.updated_at33' does NOT exist in foo saved object index patterns", isSavedObjectAttr: false, - key: 'foo.updatedAt33', + key: 'foo.updated_at33', type: 'foo', }, { diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 8fcd4cbc4f3cf6..fde2f39ee287b8 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -701,7 +701,6 @@ export class SavedObjectsRepository { * @property {Array} [options.fields] * @property {string} [options.namespace] * @property {object} [options.hasReference] - { type, id } - * @property {object} [options.aggs] - see ./saved_object_aggs for more insight * @property {string} [options.preference] * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index a9b0fd5ee9e87b..7bb919b9198c0e 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -113,8 +113,8 @@ export interface SavedObjectsFindOptions { * Specify an Elasticsearch aggregation to perform. This alpha API only supports a limited set of aggregation types: metrics, bucket. Additional aggregation types can be contributed to Core. * @alpha * @example - * aggs = { type_count: { max: { field: 'dashboard.attributes.version' } } }; - * SavedObjects.find({type: 'dashboard', aggs: '%7B%22type_count%22%3A%7B%22max%22%3A%7B%22field%22%3A%22dashboard.attributes.version%22%7D%7D%7D'}) + * aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; + * SavedObjects.find({type: 'dashboard', aggs: '%7B%22latest_version%22%3A%7B%22max%22%3A%7B%22field%22%3A%22dashboard.attributes.version%22%7D%7D%7D'}) */ aggs?: Record; namespaces?: string[]; diff --git a/test/api_integration/apis/index.js b/test/api_integration/apis/index.js index 65a388d5a52e8b..d07c099634005d 100644 --- a/test/api_integration/apis/index.js +++ b/test/api_integration/apis/index.js @@ -19,20 +19,20 @@ export default function ({ loadTestFile }) { describe('apis', () => { - // loadTestFile(require.resolve('./core')); - // loadTestFile(require.resolve('./general')); - // loadTestFile(require.resolve('./home')); - // loadTestFile(require.resolve('./index_patterns')); - // loadTestFile(require.resolve('./kql_telemetry')); - // loadTestFile(require.resolve('./saved_objects_management')); + loadTestFile(require.resolve('./core')); + loadTestFile(require.resolve('./general')); + loadTestFile(require.resolve('./home')); + loadTestFile(require.resolve('./index_patterns')); + loadTestFile(require.resolve('./kql_telemetry')); + loadTestFile(require.resolve('./saved_objects_management')); loadTestFile(require.resolve('./saved_objects')); - // loadTestFile(require.resolve('./scripts')); - // loadTestFile(require.resolve('./search')); - // loadTestFile(require.resolve('./shorten')); - // loadTestFile(require.resolve('./suggestions')); - // loadTestFile(require.resolve('./status')); - // loadTestFile(require.resolve('./stats')); - // loadTestFile(require.resolve('./ui_metric')); - // loadTestFile(require.resolve('./telemetry')); + loadTestFile(require.resolve('./scripts')); + loadTestFile(require.resolve('./search')); + loadTestFile(require.resolve('./shorten')); + loadTestFile(require.resolve('./suggestions')); + loadTestFile(require.resolve('./status')); + loadTestFile(require.resolve('./stats')); + loadTestFile(require.resolve('./ui_metric')); + loadTestFile(require.resolve('./telemetry')); }); } diff --git a/test/api_integration/apis/saved_objects/index.js b/test/api_integration/apis/saved_objects/index.js index bcdaa80b6334ec..ad6c3749181dd5 100644 --- a/test/api_integration/apis/saved_objects/index.js +++ b/test/api_integration/apis/saved_objects/index.js @@ -19,17 +19,17 @@ export default function ({ loadTestFile }) { describe('saved_objects', () => { - // loadTestFile(require.resolve('./bulk_create')); - // loadTestFile(require.resolve('./bulk_get')); - // loadTestFile(require.resolve('./create')); - // loadTestFile(require.resolve('./delete')); - // loadTestFile(require.resolve('./export')); + loadTestFile(require.resolve('./bulk_create')); + loadTestFile(require.resolve('./bulk_get')); + loadTestFile(require.resolve('./create')); + loadTestFile(require.resolve('./delete')); + loadTestFile(require.resolve('./export')); loadTestFile(require.resolve('./find')); - // loadTestFile(require.resolve('./get')); - // loadTestFile(require.resolve('./import')); - // loadTestFile(require.resolve('./resolve_import_errors')); - // loadTestFile(require.resolve('./update')); - // loadTestFile(require.resolve('./bulk_update')); - // loadTestFile(require.resolve('./migrations')); + loadTestFile(require.resolve('./get')); + loadTestFile(require.resolve('./import')); + loadTestFile(require.resolve('./resolve_import_errors')); + loadTestFile(require.resolve('./update')); + loadTestFile(require.resolve('./bulk_update')); + loadTestFile(require.resolve('./migrations')); }); } From b88bbbd35a88f3777fbf82bde7a65f0763d1f960 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 12:23:26 +0200 Subject: [PATCH 22/52] migrate added tests to new TS file --- .../apis/saved_objects/find.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index 28c38ca9e0ded1..6ff58ad019dae8 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -293,6 +293,74 @@ export default function ({ getService }: FtrProviderContext) { })); }); + describe('with a aggs', () => { + it('should return 200 with a valid response', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'visualization.attributes.version' } }, + }) + )}` + ) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + aggregations: { + type_count: { + value: 1, + }, + }, + page: 1, + per_page: 0, + saved_objects: [], + total: 1, + }); + })); + + it('wrong type should return 400 with Bad Request', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'dashboard.attributes.version' } }, + }) + )}` + ) + .expect(400) + .then((resp) => { + expect(resp.body).to.eql({ + error: 'Bad Request', + message: 'This type dashboard is not allowed: Bad Request', + statusCode: 400, + }); + })); + + it('adding a wrong attributes should return 400 with Bad Request', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { + max: { + field: 'visualization.attributes.version', + script: 'Oh yes I am going to a script', + }, + }, + }) + )}` + ) + .expect(400) + .then((resp) => { + expect(resp.body).to.eql({ + error: 'Bad Request', + message: + 'script attribute is not supported in saved objects aggregation: Bad Request', + statusCode: 400, + }); + })); + }); + describe('`has_reference` and `has_reference_operator` parameters', () => { before(() => esArchiver.load('saved_objects/references')); after(() => esArchiver.unload('saved_objects/references')); From 3682d3079060a46f51da01d9711f665fe69bfc4d Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 12:27:45 +0200 Subject: [PATCH 23/52] fix documentation --- src/core/server/saved_objects/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 56e580827330ac..2a887a1ca4ef4d 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -121,7 +121,7 @@ export interface SavedObjectsFindOptions { * @alpha * @example * aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; - * SavedObjects.find({type: 'dashboard', aggs: '%7B%22latest_version%22%3A%7B%22max%22%3A%7B%22field%22%3A%22dashboard.attributes.version%22%7D%7D%7D'}) + * SavedObjects.find({ type: 'dashboard', aggs }) */ aggs?: Record; namespaces?: string[]; From 3d375564bdbbb01b25e6627207be3fa55f47c944 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 12:38:26 +0200 Subject: [PATCH 24/52] fix license header --- src/core/server/saved_objects/routes/find.ts | 16 +++++++++++++- .../service/lib/aggs_utils.test.ts | 21 +++++-------------- .../saved_objects/service/lib/aggs_utils.ts | 21 +++++-------------- .../bucket_aggs/index.ts | 21 +++++-------------- .../lib/saved_objects_aggs_types/helpers.ts | 21 +++++-------------- .../lib/saved_objects_aggs_types/index.ts | 21 +++++-------------- .../metrics_aggs/index.ts | 21 +++++-------------- 7 files changed, 45 insertions(+), 97 deletions(-) diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index e3070eeec989c7..d21039db30e5ff 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -60,6 +60,20 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen const usageStatsClient = coreUsageData.getClient(); usageStatsClient.incrementSavedObjectsFind({ request: req }).catch(() => {}); + // manually validation to avoid using JSON.parse twice + let aggs; + if (query.aggs) { + try { + aggs = JSON.parse(query.aggs); + } catch (e) { + return res.badRequest({ + body: { + message: 'invalid aggs value', + }, + }); + } + } + const result = await context.core.savedObjects.client.find({ perPage: query.per_page, page: query.page, @@ -73,7 +87,7 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen hasReferenceOperator: query.has_reference_operator, fields: typeof query.fields === 'string' ? [query.fields] : query.fields, filter: query.filter, - aggs: query.aggs ? JSON.parse(query.aggs) : undefined, + aggs, namespaces, }); diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index a273772367a382..2a270fffe5ce3c 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { validateGetSavedObjectsAggs } from './aggs_utils'; diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index 2642da56f6b826..568e72dd9198c6 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { IndexMapping } from '../../mappings'; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts index cb9c5febce6e32..29a92a72ad4b33 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import * as rt from 'io-ts'; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts index 6485639636ec34..ca18a50cb8f509 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import * as rt from 'io-ts'; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts index a60b38ed4db453..0d8f1bfe3f1aba 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { fold } from 'fp-ts/lib/Either'; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts index 7de8ed12dcc69a..2d2ac364a72e23 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import * as rt from 'io-ts'; From fa35f939ec7c2deefdfe159ef29f2ec1f268e309 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 12:50:01 +0200 Subject: [PATCH 25/52] move stuff --- .../service/lib/aggregations/aggs_types/base.ts | 12 ++++++++++++ .../aggs_types/bucket_aggs.ts} | 3 +-- .../service/lib/aggregations/aggs_types/index.ts | 10 ++++++++++ .../aggs_types/metrics_aggs.ts} | 3 +-- .../helpers.ts => aggregations/errors.ts} | 3 --- .../index.ts | 7 ++----- .../server/saved_objects/service/lib/aggs_utils.ts | 3 +-- 7 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts rename src/core/server/saved_objects/service/lib/{saved_objects_aggs_types/bucket_aggs/index.ts => aggregations/aggs_types/bucket_aggs.ts} (96%) create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts rename src/core/server/saved_objects/service/lib/{saved_objects_aggs_types/metrics_aggs/index.ts => aggregations/aggs_types/metrics_aggs.ts} (98%) rename src/core/server/saved_objects/service/lib/{saved_objects_aggs_types/helpers.ts => aggregations/errors.ts} (85%) rename src/core/server/saved_objects/service/lib/{saved_objects_aggs_types => aggregations}/index.ts (85%) diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts new file mode 100644 index 00000000000000..b50e4a6cf5ab5c --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as rt from 'io-ts'; + +export const fieldBasic = { field: rt.string }; +export const FieldBasicRT = rt.type({ field: rt.string }); diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts similarity index 96% rename from src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts rename to src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts index 29a92a72ad4b33..e86d21f9392244 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/bucket_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -7,8 +7,7 @@ */ import * as rt from 'io-ts'; - -import { fieldBasic } from '../helpers'; +import { fieldBasic } from './base'; export const bucketAggsType: Record> = { filter: { diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts new file mode 100644 index 00000000000000..1334f7550ee89e --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { bucketAggsType } from './bucket_aggs'; +export { metricsAggsType } from './metrics_aggs'; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts similarity index 98% rename from src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts rename to src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts index 2d2ac364a72e23..980194444cd06a 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/metrics_aggs/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts @@ -7,8 +7,7 @@ */ import * as rt from 'io-ts'; - -import { fieldBasic, FieldBasicRT } from '../helpers'; +import { fieldBasic, FieldBasicRT } from './base'; /* * Types for Metrics Aggregations diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts b/src/core/server/saved_objects/service/lib/aggregations/errors.ts similarity index 85% rename from src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts rename to src/core/server/saved_objects/service/lib/aggregations/errors.ts index ca18a50cb8f509..084c6a652b829f 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/helpers.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/errors.ts @@ -11,9 +11,6 @@ import { failure } from 'io-ts/lib/PathReporter'; type ErrorFactory = (message: string) => Error; -export const fieldBasic = { field: rt.string }; -export const FieldBasicRT = rt.type({ field: rt.string }); - export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { throw createError(failure(errors).join('\n')); }; diff --git a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts b/src/core/server/saved_objects/service/lib/aggregations/index.ts similarity index 85% rename from src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts rename to src/core/server/saved_objects/service/lib/aggregations/index.ts index 0d8f1bfe3f1aba..95a06d790d7980 100644 --- a/src/core/server/saved_objects/service/lib/saved_objects_aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/index.ts @@ -9,14 +9,11 @@ import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; import { pipe } from 'fp-ts/lib/pipeable'; - import * as rt from 'io-ts'; -import { bucketAggsType } from './bucket_aggs'; -import { metricsAggsType } from './metrics_aggs'; - import { SavedObjectsErrorHelpers } from '../errors'; -import { throwErrors } from './helpers'; +import { bucketAggsType, metricsAggsType } from './aggs_types'; +import { throwErrors } from './errors'; export const savedObjectsAggs = { ...metricsAggsType, diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index 568e72dd9198c6..07049298cb2edd 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -7,10 +7,9 @@ */ import { IndexMapping } from '../../mappings'; - import { SavedObjectsErrorHelpers } from './errors'; import { hasFilterKeyError } from './filter_utils'; -import { savedObjectsAggs, validateSavedObjectsTypeAggs } from './saved_objects_aggs_types'; +import { savedObjectsAggs, validateSavedObjectsTypeAggs } from './aggregations'; export const validateGetSavedObjectsAggs = ( allowedTypes: string[], From 3efad5b66a1d92551e9e479062988ce290de940b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 12:55:04 +0200 Subject: [PATCH 26/52] duplicating test mappings --- .../service/lib/aggs_utils.test.ts | 78 ++++++++++++++++++- .../service/lib/filter_utils.test.ts | 3 +- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index 2a270fffe5ce3c..21e7370517d923 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -7,9 +7,83 @@ */ import { validateGetSavedObjectsAggs } from './aggs_utils'; -import { mockMappings } from './filter_utils.test'; -describe('Filter Utils', () => { +const mockMappings = { + properties: { + updated_at: { + type: 'date', + }, + foo: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + bytes: { + type: 'number', + }, + }, + }, + bar: { + properties: { + foo: { + type: 'text', + }, + description: { + type: 'text', + }, + }, + }, + bean: { + properties: { + canned: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + alert: { + properties: { + actions: { + type: 'nested', + properties: { + group: { + type: 'keyword', + }, + actionRef: { + type: 'keyword', + }, + actionTypeId: { + type: 'keyword', + }, + params: { + enabled: false, + type: 'object', + }, + }, + }, + params: { + type: 'flattened', + }, + }, + }, + hiddenType: { + properties: { + description: { + type: 'text', + }, + }, + }, + }, +}; + +describe('Aggregation Utils', () => { describe('#validateGetSavedObjectsAggs', () => { test('Validate a simple aggregations', () => { expect( diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index 376610060df576..956a60b23809d3 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -16,7 +16,7 @@ import { hasFilterKeyError, } from './filter_utils'; -export const mockMappings = { +const mockMappings = { properties: { updated_at: { type: 'date', @@ -529,7 +529,6 @@ describe('Filter Utils', () => { indexMapping: mockMappings, }); - // nodes will have errors in the array expect(validationObject).toEqual([ { astPath: 'arguments.1.arguments.0', From af2b74305163bc961564ce4f816ce843c0fa3ddf Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 13:11:47 +0200 Subject: [PATCH 27/52] rename some stuff --- .../service/lib/aggs_utils.test.ts | 26 +++++++++---------- .../saved_objects/service/lib/aggs_utils.ts | 10 +++---- .../saved_objects/service/lib/repository.ts | 4 +-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts index 21e7370517d923..0b944064e50e75 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { validateGetSavedObjectsAggs } from './aggs_utils'; +import { validateAndConvertAggregations } from './aggs_utils'; const mockMappings = { properties: { @@ -87,7 +87,7 @@ describe('Aggregation Utils', () => { describe('#validateGetSavedObjectsAggs', () => { test('Validate a simple aggregations', () => { expect( - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['foo'], { aggName: { max: { field: 'foo.attributes.bytes' } } }, mockMappings @@ -103,7 +103,7 @@ describe('Aggregation Utils', () => { test('Validate a nested field in simple aggregations', () => { expect( - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, mockMappings @@ -119,7 +119,7 @@ describe('Aggregation Utils', () => { test('Validate a nested aggregations', () => { expect( - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { @@ -153,7 +153,7 @@ describe('Aggregation Utils', () => { test('Validate an aggregation without the attribute field', () => { expect( - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { terms: { 'alert.attributes.actions.group': ['myFriend', 'snoopy'] } } }, mockMappings @@ -169,7 +169,7 @@ describe('Aggregation Utils', () => { test('Validate a filter term aggregations', () => { expect( - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['foo'], { aggName: { filter: { term: { 'foo.attributes.bytes': 10 } } } }, mockMappings @@ -185,7 +185,7 @@ describe('Aggregation Utils', () => { test('Throw an error when types is not allowed', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { @@ -199,7 +199,7 @@ describe('Aggregation Utils', () => { test('Throw an error when add an invalid attributes ', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['foo'], { aggName: { @@ -215,7 +215,7 @@ describe('Aggregation Utils', () => { test('Throw an error when an attributes is not defined correctly', () => { expect(() => - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { @@ -229,7 +229,7 @@ describe('Aggregation Utils', () => { test('Throw an error when aggregation is not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['foo'], { aggName: { @@ -245,7 +245,7 @@ describe('Aggregation Utils', () => { test('Throw an error when children aggregation is not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['foo'], { aggName: { @@ -268,7 +268,7 @@ describe('Aggregation Utils', () => { test('Throw an error when you add the script attribute who are not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { @@ -285,7 +285,7 @@ describe('Aggregation Utils', () => { test('Throw an error when you add the script attribute in a nested aggregations who are not defined in SavedObjectsAggs', () => { expect(() => { - validateGetSavedObjectsAggs( + validateAndConvertAggregations( ['alert'], { aggName: { diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggs_utils.ts index 07049298cb2edd..718540e9695f8b 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggs_utils.ts @@ -11,15 +11,15 @@ import { SavedObjectsErrorHelpers } from './errors'; import { hasFilterKeyError } from './filter_utils'; import { savedObjectsAggs, validateSavedObjectsTypeAggs } from './aggregations'; -export const validateGetSavedObjectsAggs = ( +export const validateAndConvertAggregations = ( allowedTypes: string[], aggs: Record, indexMapping: IndexMapping ) => { - return validateGetAggFieldValue(allowedTypes, aggs, indexMapping); + return validateAggFieldValue(allowedTypes, aggs, indexMapping); }; -const validateGetAggFieldValue = ( +const validateAggFieldValue = ( allowedTypes: string[], aggs: any, indexMapping: IndexMapping, @@ -35,7 +35,7 @@ const validateGetAggFieldValue = ( if (typeof aggs[key] === 'object' && aggType === undefined && savedObjectsAggs[key]) { return { ...acc, - [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping, key, key), + [key]: validateAggFieldValue(allowedTypes, aggs[key], indexMapping, key, key), }; } else if ( typeof aggs[key] === 'object' && @@ -43,7 +43,7 @@ const validateGetAggFieldValue = ( ) { return { ...acc, - [key]: validateGetAggFieldValue(allowedTypes, aggs[key], indexMapping, key, undefined), + [key]: validateAggFieldValue(allowedTypes, aggs[key], indexMapping, key, undefined), }; } else if ( key !== 'field' && diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 7ecff038945db5..f26481cf2fc4cb 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -66,7 +66,7 @@ import { import { LegacyUrlAlias, LEGACY_URL_ALIAS_TYPE } from '../../object_types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; -import { validateGetSavedObjectsAggs } from './aggs_utils'; +import { validateAndConvertAggregations } from './aggs_utils'; import { ALL_NAMESPACES_STRING, FIND_DEFAULT_PAGE, @@ -830,7 +830,7 @@ export class SavedObjectsRepository { let aggsObject = null; if (aggs) { try { - aggsObject = validateGetSavedObjectsAggs(allowedTypes, aggs, this._mappings); + aggsObject = validateAndConvertAggregations(allowedTypes, aggs, this._mappings); } catch (e) { throw e; } From 2bb16d00e20b313a7df5e2df917db04c721fc896 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 13:29:55 +0200 Subject: [PATCH 28/52] move ALL the things --- .../lib/aggregations/aggs_types/index.ts | 9 +++- .../service/lib/aggregations/index.ts | 21 +--------- .../validation.test.ts} | 2 +- .../validation.ts} | 42 ++++++++++++------- .../saved_objects/service/lib/repository.ts | 2 +- 5 files changed, 37 insertions(+), 39 deletions(-) rename src/core/server/saved_objects/service/lib/{aggs_utils.test.ts => aggregations/validation.test.ts} (99%) rename src/core/server/saved_objects/service/lib/{aggs_utils.ts => aggregations/validation.ts} (63%) diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts index 1334f7550ee89e..1f93c20a14480d 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts @@ -6,5 +6,10 @@ * Side Public License, v 1. */ -export { bucketAggsType } from './bucket_aggs'; -export { metricsAggsType } from './metrics_aggs'; +import { bucketAggsType } from './bucket_aggs'; +import { metricsAggsType } from './metrics_aggs'; + +export const aggsTypes = { + ...metricsAggsType, + ...bucketAggsType, +}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/index.ts b/src/core/server/saved_objects/service/lib/aggregations/index.ts index 95a06d790d7980..f71d3e8daea9d2 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/index.ts @@ -6,23 +6,4 @@ * Side Public License, v 1. */ -import { fold } from 'fp-ts/lib/Either'; -import { identity } from 'fp-ts/lib/function'; -import { pipe } from 'fp-ts/lib/pipeable'; -import * as rt from 'io-ts'; - -import { SavedObjectsErrorHelpers } from '../errors'; -import { bucketAggsType, metricsAggsType } from './aggs_types'; -import { throwErrors } from './errors'; - -export const savedObjectsAggs = { - ...metricsAggsType, - ...bucketAggsType, -}; - -export const validateSavedObjectsTypeAggs = (rtType: rt.Any, aggObject: unknown) => { - pipe( - rtType.decode(aggObject), - fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) - ); -}; +export { validateAndConvertAggregations } from './validation'; diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts similarity index 99% rename from src/core/server/saved_objects/service/lib/aggs_utils.test.ts rename to src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index 0b944064e50e75..d6ca72c0613047 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { validateAndConvertAggregations } from './aggs_utils'; +import { validateAndConvertAggregations } from './validation'; const mockMappings = { properties: { diff --git a/src/core/server/saved_objects/service/lib/aggs_utils.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts similarity index 63% rename from src/core/server/saved_objects/service/lib/aggs_utils.ts rename to src/core/server/saved_objects/service/lib/aggregations/validation.ts index 718540e9695f8b..3d92554a411062 100644 --- a/src/core/server/saved_objects/service/lib/aggs_utils.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -6,20 +6,25 @@ * Side Public License, v 1. */ -import { IndexMapping } from '../../mappings'; -import { SavedObjectsErrorHelpers } from './errors'; -import { hasFilterKeyError } from './filter_utils'; -import { savedObjectsAggs, validateSavedObjectsTypeAggs } from './aggregations'; +import * as rt from 'io-ts'; +import { pipe } from 'fp-ts/pipeable'; +import { fold } from 'fp-ts/Either'; +import { identity } from 'fp-ts/function'; +import { throwErrors } from './errors'; +import { IndexMapping } from '../../../mappings'; +import { SavedObjectsErrorHelpers } from '../errors'; +import { hasFilterKeyError } from '../filter_utils'; +import { aggsTypes } from './aggs_types'; export const validateAndConvertAggregations = ( allowedTypes: string[], aggs: Record, indexMapping: IndexMapping ) => { - return validateAggFieldValue(allowedTypes, aggs, indexMapping); + return recurseNestedValidation(allowedTypes, aggs, indexMapping); }; -const validateAggFieldValue = ( +const recurseNestedValidation = ( allowedTypes: string[], aggs: any, indexMapping: IndexMapping, @@ -32,10 +37,10 @@ const validateAggFieldValue = ( 'script attribute is not supported in saved objects aggregation' ); } - if (typeof aggs[key] === 'object' && aggType === undefined && savedObjectsAggs[key]) { + if (typeof aggs[key] === 'object' && aggType === undefined && aggsTypes[key]) { return { ...acc, - [key]: validateAggFieldValue(allowedTypes, aggs[key], indexMapping, key, key), + [key]: recurseNestedValidation(allowedTypes, aggs[key], indexMapping, key, key), }; } else if ( typeof aggs[key] === 'object' && @@ -43,21 +48,21 @@ const validateAggFieldValue = ( ) { return { ...acc, - [key]: validateAggFieldValue(allowedTypes, aggs[key], indexMapping, key, undefined), + [key]: recurseNestedValidation(allowedTypes, aggs[key], indexMapping, key, undefined), }; } else if ( key !== 'field' && aggType && - savedObjectsAggs[aggType] !== undefined && - savedObjectsAggs[aggType][key] !== undefined + aggsTypes[aggType] !== undefined && + aggsTypes[aggType][key] !== undefined ) { - validateSavedObjectsTypeAggs(savedObjectsAggs[aggType][key], aggs[key]); + validateFieldValue(aggsTypes[aggType][key], aggs[key]); return { ...acc, [key]: aggs[key], }; } else { - if (aggType === undefined || savedObjectsAggs[aggType] === undefined) { + if (aggType === undefined || aggsTypes[aggType] === undefined) { throw SavedObjectsErrorHelpers.createBadRequestError( `This aggregation ${lastKey} is not valid or we did not defined it yet` ); @@ -70,8 +75,8 @@ const validateAggFieldValue = ( if (error != null) { if ( aggType !== undefined && - savedObjectsAggs[aggType] !== undefined && - savedObjectsAggs[aggType][key] === undefined + aggsTypes[aggType] !== undefined && + aggsTypes[aggType][key] === undefined ) { throw SavedObjectsErrorHelpers.createBadRequestError( `${key} attribute is not supported in ${aggType} saved objects aggregation` @@ -88,3 +93,10 @@ const validateAggFieldValue = ( } }, {}); }; + +const validateFieldValue = (rtType: rt.Any, aggObject: unknown) => { + pipe( + rtType.decode(aggObject), + fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) + ); +}; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index f26481cf2fc4cb..f001885946ce70 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -66,7 +66,7 @@ import { import { LegacyUrlAlias, LEGACY_URL_ALIAS_TYPE } from '../../object_types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; -import { validateAndConvertAggregations } from './aggs_utils'; +import { validateAndConvertAggregations } from './aggregations'; import { ALL_NAMESPACES_STRING, FIND_DEFAULT_PAGE, From e150741c561763327740ea7c6e07d47a314d1e13 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 13:34:40 +0200 Subject: [PATCH 29/52] cast to aggregation container --- .../saved_objects/service/lib/aggregations/validation.ts | 8 ++++++-- src/core/server/saved_objects/service/lib/repository.ts | 2 +- src/core/server/saved_objects/types.ts | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index 3d92554a411062..5d794752277ffe 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -10,6 +10,7 @@ import * as rt from 'io-ts'; import { pipe } from 'fp-ts/pipeable'; import { fold } from 'fp-ts/Either'; import { identity } from 'fp-ts/function'; +import type { estypes } from '@elastic/elasticsearch'; import { throwErrors } from './errors'; import { IndexMapping } from '../../../mappings'; import { SavedObjectsErrorHelpers } from '../errors'; @@ -20,8 +21,11 @@ export const validateAndConvertAggregations = ( allowedTypes: string[], aggs: Record, indexMapping: IndexMapping -) => { - return recurseNestedValidation(allowedTypes, aggs, indexMapping); +): Record => { + return recurseNestedValidation(allowedTypes, aggs, indexMapping) as Record< + string, + estypes.AggregationContainer + >; }; const recurseNestedValidation = ( diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index f001885946ce70..d867f931a1b134 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -885,7 +885,7 @@ export class SavedObjectsRepository { } return { - ...(body.aggregations != null ? { aggregations: body.aggregations } : {}), + ...(body.aggregations != null ? { aggregations: body.aggregations as any } : {}), page, per_page: perPage, total: body.hits.total, diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 2a887a1ca4ef4d..08ac6cbc15b585 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -123,7 +123,7 @@ export interface SavedObjectsFindOptions { * aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; * SavedObjects.find({ type: 'dashboard', aggs }) */ - aggs?: Record; + aggs?: Record; namespaces?: string[]; /** * This map defines each type to search for, and the namespace(s) to search for the type in; this is only intended to be used by a saved From f60cb6f87d43c420c92bfb81872cac446a53e79b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 13:38:34 +0200 Subject: [PATCH 30/52] update generated doc --- src/core/public/public.api.md | 2 +- src/core/server/server.api.md | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 20c7eeb92f7781..ddb15e201cd718 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1233,7 +1233,7 @@ export interface SavedObjectsCreateOptions { // @public (undocumented) export interface SavedObjectsFindOptions { // @alpha - aggs?: Record; + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 53b2eb86104183..9aab26a64bf5dc 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2244,7 +2244,7 @@ export class SavedObjectsClient { static errors: typeof SavedObjectsErrorHelpers; // (undocumented) errors: typeof SavedObjectsErrorHelpers; - find(options: SavedObjectsFindOptions): Promise>; + find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; openPointInTimeForType(type: string | string[], options?: SavedObjectsOpenPointInTimeOptions): Promise; removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise; @@ -2501,6 +2501,8 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions { + // @alpha + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts @@ -2539,7 +2541,9 @@ export interface SavedObjectsFindOptionsReference { } // @public -export interface SavedObjectsFindResponse { +export interface SavedObjectsFindResponse { + // (undocumented) + aggregations?: A; // (undocumented) page: number; // (undocumented) @@ -2849,7 +2853,7 @@ export class SavedObjectsRepository { deleteByNamespace(namespace: string, options?: SavedObjectsDeleteByNamespaceOptions): Promise; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise; // (undocumented) - find(options: SavedObjectsFindOptions): Promise>; + find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>; openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise; @@ -2970,7 +2974,7 @@ export interface SavedObjectsUpdateResponse extends Omit({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; + static createEmptyFindResponse: ({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; static generateId(): string; static isRandomId(id: string | undefined): boolean; static namespaceIdToString: (namespace?: string | undefined) => string; From 2efb54c535f7910930588169bd0ea7cda43bc795 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 6 Apr 2021 13:48:49 +0200 Subject: [PATCH 31/52] add deep nested validation --- .../lib/aggregations/validation.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index d6ca72c0613047..7e53a6746473df 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -151,6 +151,52 @@ describe('Aggregation Utils', () => { }); }); + test('Validate a deeply nested aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { + first: { + cardinality: { + field: 'alert.attributes.actions.group', + aggs: { + second: { + max: { field: 'alert.attributes.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.attributes.actions.actionTypeId', + }, + }, + }, + }, + }, + }, + }, + }, + mockMappings + ) + ).toEqual({ + first: { + cardinality: { + field: 'alert.actions.group', + aggs: { + second: { + max: { field: 'alert.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.actions.actionTypeId', + }, + }, + }, + }, + }, + }, + }, + }); + }); + test('Validate an aggregation without the attribute field', () => { expect( validateAndConvertAggregations( From 4adf27d62e8d16ac83e2e9c49ac6c67cf25d5d30 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 11:47:49 +0200 Subject: [PATCH 32/52] rewrite the whole validation mechanism --- .../lib/aggregations/aggs_types/base.ts | 12 - .../aggregations/aggs_types/bucket_aggs.ts | 69 ++-- .../lib/aggregations/aggs_types/index.ts | 10 +- .../aggregations/aggs_types/metrics_aggs.ts | 87 +++-- .../service/lib/aggregations/errors.ts | 16 - .../lib/aggregations/validation.test.ts | 339 ++++++++++-------- .../service/lib/aggregations/validation.ts | 292 ++++++++++----- 7 files changed, 506 insertions(+), 319 deletions(-) delete mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts delete mode 100644 src/core/server/saved_objects/service/lib/aggregations/errors.ts diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts deleted file mode 100644 index b50e4a6cf5ab5c..00000000000000 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/base.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import * as rt from 'io-ts'; - -export const fieldBasic = { field: rt.string }; -export const FieldBasicRT = rt.type({ field: rt.string }); diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts index e86d21f9392244..1c98653aa6720f 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -6,32 +6,47 @@ * Side Public License, v 1. */ -import * as rt from 'io-ts'; -import { fieldBasic } from './base'; +import { schema as s, ObjectType } from '@kbn/config-schema'; -export const bucketAggsType: Record> = { - filter: { - term: rt.record(rt.string, rt.any), - }, - histogram: { - ...fieldBasic, - interval: rt.number, - min_doc_count: rt.number, - extended_bounds: rt.type({ min: rt.number, max: rt.number }), - keyed: rt.boolean, - missing: rt.number, - order: rt.record(rt.string, rt.literal('asc', 'desc')), - }, - terms: { - ...fieldBasic, - collect_mode: rt.string, - exclude: rt.unknown, - execution_hint: rt.string, - include: rt.unknown, - missing: rt.string, - min_doc_count: rt.number, - size: rt.number, - show_term_doc_count_error: rt.boolean, - order: rt.record(rt.string, rt.literal('asc', 'desc')), - }, +export const bucketAggsSchemas: Record = { + filter: s.object({ + term: s.recordOf(s.string(), s.oneOf([s.string(), s.boolean(), s.number()])), + }), + histogram: s.object({ + field: s.maybe(s.string()), + interval: s.maybe(s.number()), + min_doc_count: s.maybe(s.number()), + extended_bounds: s.maybe( + s.object({ + min: s.number(), + max: s.number(), + }) + ), + hard_bounds: s.maybe( + s.object({ + min: s.number(), + max: s.number(), + }) + ), + missing: s.maybe(s.number()), + keyed: s.maybe(s.boolean()), + order: s.maybe( + s.object({ + _count: s.string(), + _key: s.string(), + }) + ), + }), + terms: s.object({ + field: s.maybe(s.string()), + collect_mode: s.maybe(s.string()), + exclude: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + include: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + execution_hint: s.maybe(s.string()), + missing: s.maybe(s.number()), + min_doc_count: s.maybe(s.number()), + size: s.maybe(s.number()), + show_term_doc_count_error: s.maybe(s.boolean()), + order: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + }), }; diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts index 1f93c20a14480d..7967fad0185fbb 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -import { bucketAggsType } from './bucket_aggs'; -import { metricsAggsType } from './metrics_aggs'; +import { bucketAggsSchemas } from './bucket_aggs'; +import { metricsAggsSchemas } from './metrics_aggs'; -export const aggsTypes = { - ...metricsAggsType, - ...bucketAggsType, +export const aggregationSchemas = { + ...metricsAggsSchemas, + ...bucketAggsSchemas, }; diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts index 980194444cd06a..85dec9614675cd 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts @@ -6,13 +6,12 @@ * Side Public License, v 1. */ -import * as rt from 'io-ts'; -import { fieldBasic, FieldBasicRT } from './base'; +import { schema as s, ObjectType } from '@kbn/config-schema'; /* * Types for Metrics Aggregations * - * TODO: + * Not implemented: * - Extended Stats Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-extendedstats-aggregation.html * - Geo Bounds Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geobounds-aggregation.html * - Geo Centroid Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geocentroid-aggregation.html @@ -27,36 +26,54 @@ import { fieldBasic, FieldBasicRT } from './base'; * - Median Absolute Deviation Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-median-absolute-deviation-aggregation.html */ -export const metricsAggsType: Record> = { - avg: fieldBasic, - weighted_avg: { - value: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), - weight: rt.intersection([FieldBasicRT, rt.partial({ missing: rt.number })]), - format: rt.string, - value_type: rt.string, - }, - cardinality: fieldBasic, - max: { - ...fieldBasic, - missing: rt.number, - }, - min: { - ...fieldBasic, - missing: rt.number, - }, - top_hits: { - explain: rt.boolean, - from: rt.string, - highlight: rt.any, - seq_no_primary_term: rt.boolean, - size: rt.number, - sort: rt.any, - stored_fields: rt.array(rt.string), - version: rt.boolean, - _name: rt.string, - _source: rt.partial({ - includes: rt.array(rt.string), - excludes: rt.array(rt.string), - }), - }, +export const metricsAggsSchemas: Record = { + avg: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + cardinality: s.object({ + field: s.maybe(s.string()), + precision_threshold: s.maybe(s.number()), + rehash: s.maybe(s.boolean()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + min: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + format: s.maybe(s.string()), + }), + max: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + format: s.maybe(s.string()), + }), + weighted_avg: s.object({ + format: s.maybe(s.string()), + value_type: s.maybe(s.string()), + value: s.maybe( + s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.number()), + }) + ), + weight: s.maybe( + s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.number()), + }) + ), + }), + top_hits: s.object({ + explain: s.maybe(s.boolean()), + docvalue_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + from: s.maybe(s.number()), + size: s.maybe(s.number()), + sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + seq_no_primary_term: s.maybe(s.boolean()), + version: s.maybe(s.boolean()), + track_scores: s.maybe(s.boolean()), + highlight: s.maybe(s.any()), + _source: s.maybe(s.oneOf([s.boolean(), s.string(), s.arrayOf(s.string())])), + }), }; diff --git a/src/core/server/saved_objects/service/lib/aggregations/errors.ts b/src/core/server/saved_objects/service/lib/aggregations/errors.ts deleted file mode 100644 index 084c6a652b829f..00000000000000 --- a/src/core/server/saved_objects/service/lib/aggregations/errors.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import * as rt from 'io-ts'; -import { failure } from 'io-ts/lib/PathReporter'; - -type ErrorFactory = (message: string) => Error; - -export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { - throw createError(failure(errors).join('\n')); -}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index 7e53a6746473df..4bad2d92140fe9 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -6,8 +6,11 @@ * Side Public License, v 1. */ +import type { estypes } from '@elastic/elasticsearch'; import { validateAndConvertAggregations } from './validation'; +type AggsMap = Record; + const mockMappings = { properties: { updated_at: { @@ -85,7 +88,7 @@ const mockMappings = { describe('Aggregation Utils', () => { describe('#validateGetSavedObjectsAggs', () => { - test('Validate a simple aggregations', () => { + it('Validates a simple aggregations', () => { expect( validateAndConvertAggregations( ['foo'], @@ -101,7 +104,7 @@ describe('Aggregation Utils', () => { }); }); - test('Validate a nested field in simple aggregations', () => { + it('validates a nested field in simple aggregations', () => { expect( validateAndConvertAggregations( ['alert'], @@ -117,7 +120,7 @@ describe('Aggregation Utils', () => { }); }); - test('Validate a nested aggregations', () => { + it('validates a nested aggregations', () => { expect( validateAndConvertAggregations( ['alert'], @@ -125,10 +128,10 @@ describe('Aggregation Utils', () => { aggName: { cardinality: { field: 'alert.attributes.actions.group', - aggs: { - aggName: { - max: { field: 'alert.attributes.actions.group' }, - }, + }, + aggs: { + aggName: { + max: { field: 'alert.attributes.actions.group' }, }, }, }, @@ -139,11 +142,11 @@ describe('Aggregation Utils', () => { aggName: { cardinality: { field: 'alert.actions.group', - aggs: { - aggName: { - max: { - field: 'alert.actions.group', - }, + }, + aggs: { + aggName: { + max: { + field: 'alert.actions.group', }, }, }, @@ -151,7 +154,7 @@ describe('Aggregation Utils', () => { }); }); - test('Validate a deeply nested aggregations', () => { + it('validates a deeply nested aggregations', () => { expect( validateAndConvertAggregations( ['alert'], @@ -159,14 +162,14 @@ describe('Aggregation Utils', () => { first: { cardinality: { field: 'alert.attributes.actions.group', - aggs: { - second: { - max: { field: 'alert.attributes.actions.group' }, - aggs: { - third: { - min: { - field: 'alert.attributes.actions.actionTypeId', - }, + }, + aggs: { + second: { + max: { field: 'alert.attributes.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.attributes.actions.actionTypeId', }, }, }, @@ -180,14 +183,14 @@ describe('Aggregation Utils', () => { first: { cardinality: { field: 'alert.actions.group', - aggs: { - second: { - max: { field: 'alert.actions.group' }, - aggs: { - third: { - min: { - field: 'alert.actions.actionTypeId', - }, + }, + aggs: { + second: { + max: { field: 'alert.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.actions.actionTypeId', }, }, }, @@ -197,161 +200,213 @@ describe('Aggregation Utils', () => { }); }); - test('Validate an aggregation without the attribute field', () => { - expect( - validateAndConvertAggregations( - ['alert'], - { aggName: { terms: { 'alert.attributes.actions.group': ['myFriend', 'snoopy'] } } }, - mockMappings - ) - ).toEqual({ - aggName: { - terms: { - 'alert.actions.group': ['myFriend', 'snoopy'], + it('rewrites the `field` name when valid', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.group', + missing: 10, + }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + avg: { + field: 'alert.actions.group', + missing: 10, }, }, }); }); - test('Validate a filter term aggregations', () => { - expect( - validateAndConvertAggregations( - ['foo'], - { aggName: { filter: { term: { 'foo.attributes.bytes': 10 } } } }, - mockMappings - ) - ).toEqual({ - aggName: { - filter: { - term: { 'foo.attributes.bytes': 10 }, + it('throws an error when the `field` name is not using attributes path', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.actions.group', + missing: 10, }, }, - }); + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"This key 'alert.actions.group' does NOT match the filter proposition SavedObjectType.attributes.key"` + ); }); - test('Throw an error when types is not allowed', () => { - expect(() => { - validateAndConvertAggregations( - ['alert'], - { - aggName: { - max: { field: 'foo.attributes.bytes' }, - }, + it('throws an error when the `field` name is referencing an invalid field', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.non_existing', + missing: 10, }, - mockMappings - ); - }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed: Bad Request"`); + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"This key 'alert.attributes.actions.non_existing' does NOT exist in alert saved object index patterns"` + ); }); - test('Throw an error when add an invalid attributes ', () => { - expect(() => { - validateAndConvertAggregations( - ['foo'], - { - aggName: { - max: { field: 'foo.attributes.bytes', notValid: 'yesIamNotValid' }, + it('rewrites the `field` name even when nested', () => { + const aggregations: AggsMap = { + average: { + weighted_avg: { + value: { + field: 'alert.attributes.actions.group', + missing: 10, + }, + weight: { + field: 'alert.attributes.actions.actionRef', }, }, - mockMappings - ); - }).toThrowErrorMatchingInlineSnapshot( - `"notValid attribute is not supported in max saved objects aggregation: Bad Request"` - ); + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + weighted_avg: { + value: { + field: 'alert.actions.group', + missing: 10, + }, + weight: { + field: 'alert.actions.actionRef', + }, + }, + }, + }); }); - test('Throw an error when an attributes is not defined correctly', () => { - expect(() => - validateAndConvertAggregations( - ['alert'], - { - aggName: { - terms: { 'alert.attributes.actions.group': ['myFriend', 'snoopy'], missing: 0 }, + it('rewrites the entries of a filter term record', () => { + const aggregations: AggsMap = { + myFilter: { + filter: { + term: { + 'foo.attributes.description': 'hello', + 'foo.attributes.bytes': 10, }, }, - mockMappings - ) - ).toThrowErrorMatchingInlineSnapshot(`"Invalid value 0 supplied to : string: Bad Request"`); + }, + }; + expect(validateAndConvertAggregations(['foo'], aggregations, mockMappings)).toEqual({ + myFilter: { + filter: { + term: { 'foo.description': 'hello', 'foo.bytes': 10 }, + }, + }, + }); }); - test('Throw an error when aggregation is not defined in SavedObjectsAggs', () => { + it('throws an error when referencing non-allowed types', () => { + const aggregations: AggsMap = { + myFilter: { + max: { + field: 'foo.attributes.bytes', + }, + }, + }; + expect(() => { - validateAndConvertAggregations( - ['foo'], - { - aggName: { - MySuperAgg: { field: 'foo.attributes.bytes' }, - }, + validateAndConvertAggregations(['alert'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed"`); + }); + + it('throws an error when an attributes is not respecting its schema definition', () => { + const aggregations: AggsMap = { + someAgg: { + terms: { + missing: 'expecting a number', }, - mockMappings - ); - }).toThrowErrorMatchingInlineSnapshot( - `"This aggregation MySuperAgg is not valid or we did not defined it yet: Bad Request"` + }, + }; + + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.terms.missing]: expected value of type [number] but got [string]"` ); }); - test('Throw an error when children aggregation is not defined in SavedObjectsAggs', () => { + it('throws an error when trying to validate an unknown aggregation type', () => { + const aggregations: AggsMap = { + someAgg: { + auto_date_histogram: { + field: 'foo.attributes.bytes', + }, + }, + }; + expect(() => { - validateAndConvertAggregations( - ['foo'], - { - aggName: { - cardinality: { - field: 'foo.attributes.bytes', - aggs: { - aggName: { - MySuperAgg: { field: 'alert.attributes.actions.group' }, - }, - }, + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"auto_date_histogram aggregation is not valid (or not registered yet)"` + ); + }); + + it('throws an error when a child aggregation is unknown', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + }, + aggs: { + unknownAgg: { + cumulative_cardinality: { + format: 'format', }, }, }, - mockMappings - ); + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); }).toThrowErrorMatchingInlineSnapshot( - `"This aggregation MySuperAgg is not valid or we did not defined it yet: Bad Request"` + `"cumulative_cardinality aggregation is not valid (or not registered yet)"` ); }); - test('Throw an error when you add the script attribute who are not defined in SavedObjectsAggs', () => { - expect(() => { - validateAndConvertAggregations( - ['alert'], - { - aggName: { - cardinality: { field: 'alert.attributes.actions.group' }, - script: 'I want to access that I should not', - }, + it('throws an error when using a script attribute', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', }, - mockMappings - ); + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); }).toThrowErrorMatchingInlineSnapshot( - `"script attribute is not supported in saved objects aggregation: Bad Request"` + `"[someAgg.max.script]: definition for this key is missing"` ); }); - test('Throw an error when you add the script attribute in a nested aggregations who are not defined in SavedObjectsAggs', () => { - expect(() => { - validateAndConvertAggregations( - ['alert'], - { - aggName: { - cardinality: { - field: 'alert.attributes.actions.group', - aggs: { - aggName: { - max: { - field: 'alert.attributes.actions.group', - script: 'I want to access that I should not', - }, - }, - }, + it('throws an error when using a script attribute in a nested aggregation', () => { + const aggregations: AggsMap = { + someAgg: { + min: { + field: 'foo.attributes.bytes', + }, + aggs: { + nested: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', }, }, }, - mockMappings - ); + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); }).toThrowErrorMatchingInlineSnapshot( - `"script attribute is not supported in saved objects aggregation: Bad Request"` + `"[someAgg.aggs.nested.max.script]: definition for this key is missing"` ); }); }); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index 5d794752277ffe..e651909f0370b4 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -6,101 +6,229 @@ * Side Public License, v 1. */ -import * as rt from 'io-ts'; -import { pipe } from 'fp-ts/pipeable'; -import { fold } from 'fp-ts/Either'; -import { identity } from 'fp-ts/function'; import type { estypes } from '@elastic/elasticsearch'; -import { throwErrors } from './errors'; +import { ObjectType } from '@kbn/config-schema'; +import { isPlainObject } from 'lodash'; + import { IndexMapping } from '../../../mappings'; -import { SavedObjectsErrorHelpers } from '../errors'; import { hasFilterKeyError } from '../filter_utils'; -import { aggsTypes } from './aggs_types'; +import { aggregationSchemas } from './aggs_types'; + +const aggregationKeys = ['aggs', 'aggregations']; export const validateAndConvertAggregations = ( allowedTypes: string[], aggs: Record, indexMapping: IndexMapping ): Record => { - return recurseNestedValidation(allowedTypes, aggs, indexMapping) as Record< - string, - estypes.AggregationContainer - >; + return validateAggregations(aggs as any, { + // TODO: fix type + allowedTypes, + indexMapping, + currentPath: [], + }); }; -const recurseNestedValidation = ( - allowedTypes: string[], - aggs: any, - indexMapping: IndexMapping, - lastKey?: string, - aggType?: string -): unknown => { - return Object.keys(aggs).reduce((acc, key) => { - if (key === 'script') { - throw SavedObjectsErrorHelpers.createBadRequestError( - 'script attribute is not supported in saved objects aggregation' - ); - } - if (typeof aggs[key] === 'object' && aggType === undefined && aggsTypes[key]) { - return { - ...acc, - [key]: recurseNestedValidation(allowedTypes, aggs[key], indexMapping, key, key), - }; - } else if ( - typeof aggs[key] === 'object' && - (['aggs', 'aggregations'].includes(key) || aggType === undefined) - ) { - return { - ...acc, - [key]: recurseNestedValidation(allowedTypes, aggs[key], indexMapping, key, undefined), - }; - } else if ( - key !== 'field' && - aggType && - aggsTypes[aggType] !== undefined && - aggsTypes[aggType][key] !== undefined - ) { - validateFieldValue(aggsTypes[aggType][key], aggs[key]); - return { - ...acc, - [key]: aggs[key], - }; - } else { - if (aggType === undefined || aggsTypes[aggType] === undefined) { - throw SavedObjectsErrorHelpers.createBadRequestError( - `This aggregation ${lastKey} is not valid or we did not defined it yet` - ); - } - const error = hasFilterKeyError( - key === 'field' ? aggs[key] : key, - allowedTypes, - indexMapping - ); - if (error != null) { - if ( - aggType !== undefined && - aggsTypes[aggType] !== undefined && - aggsTypes[aggType][key] === undefined - ) { - throw SavedObjectsErrorHelpers.createBadRequestError( - `${key} attribute is not supported in ${aggType} saved objects aggregation` - ); - } - throw SavedObjectsErrorHelpers.createBadRequestError(error); - } - return { - ...acc, - ...(key === 'field' - ? { [key]: aggs[key].replace('.attributes', '') } - : { [key.replace('.attributes', '')]: aggs[key] }), - }; +interface ValidationContext { + allowedTypes: string[]; + indexMapping: IndexMapping; + currentPath: string[]; +} + +/** + * Validates a record of aggregation containers, + * Which can either be the root level aggregations (`SearchRequest.body.aggs`) + * Or a nested record of aggregation (`SearchRequest.body.aggs.myAggregation.aggs`) + * + * @param aggregations + * @param context + */ +const validateAggregations = ( + aggregations: Record, + context: ValidationContext +) => { + // console.log('validateAggregations', Object.keys(aggregations)); + + return Object.entries(aggregations).reduce((memo, [aggrName, aggrContainer]) => { + return { + ...memo, + [aggrName]: validateAggregation(aggrContainer, childContext(context, aggrName)), + }; + }, {}); +}; + +const childContext = (context: ValidationContext, path: string): ValidationContext => { + return { + ...context, + currentPath: [...context.currentPath, path], + }; +}; + +/** + * Validates an aggregation container, e.g an entry of `SearchRequest.body.aggs`, or + * from a nested aggregation record. + * + * @param aggregation + * @param context + */ +const validateAggregation = ( + aggregation: estypes.AggregationContainer, + context: ValidationContext +) => { + const container = validateAggregationContainer(aggregation, context); + + if (aggregation.aggregations) { + container.aggregations = validateAggregations( + aggregation.aggregations, + childContext(context, 'aggregations') + ); + } + if (aggregation.aggs) { + container.aggs = validateAggregations(aggregation.aggs, childContext(context, 'aggs')); + } + + return container; +}; + +const validateAggregationContainer = ( + container: estypes.AggregationContainer, + context: ValidationContext +) => { + return Object.entries(container).reduce((memo, [aggName, aggregation]) => { + if (aggregationKeys.includes(aggName)) { + return memo; } + return { + ...memo, + [aggName]: validateAggregationType(aggName, aggregation, childContext(context, aggName)), + }; + }, {} as estypes.AggregationContainer); +}; + +const validateAggregationType = ( + aggregationType: string, + aggregation: Record, + context: ValidationContext +) => { + const aggregationSchema = aggregationSchemas[aggregationType]; + if (!aggregationSchema) { + throw new Error(`${aggregationType} aggregation is not valid (or not registered yet)`); + } + + // console.log('*** validateAggregationType', aggregationType, aggregation); + + validateAggregationStructure(aggregationSchema, aggregation, context); + return validateAndRewriteFieldAttributes(aggregation, context); +}; + +/** + * Validate an aggregation structure against its declared schema. + */ +const validateAggregationStructure = ( + schema: ObjectType, + aggObject: unknown, + context: ValidationContext +) => { + return schema.validate(aggObject, {}, context.currentPath.join('.')); +}; + +///// +/** + * List of fields that have an attribute path as value + * + * @example + * ```ts + * avg: { + * field: 'alert.attributes.actions.group', + * }, + * ``` + */ +const attributeFields = ['field']; +/** + * List of fields that have a Record as value + * + * @example + * ```ts + * filter: { + * term: { + * 'alert.attributes.actions.group': 'value' + * }, + * }, + * ``` + */ +const attributeMaps = ['term']; + +const validateAndRewriteFieldAttributes = ( + aggregation: Record, + context: ValidationContext +) => { + return recursiveRewrite(aggregation, context, []); +}; + +const recursiveRewrite = ( + currentLevel: Record, + context: ValidationContext, + parents: string[] +): Record => { + return Object.entries(currentLevel).reduce((memo, [key, value]) => { + const rewriteKey = isAttributeKey(parents); + const rewriteValue = isAttributeValue(key, value); + + const newKey = rewriteKey ? validateAndRewriteKey(key, context) : key; + const newValue = rewriteValue + ? validateAndRewriteValue(key, value, context) + : isPlainObject(value) + ? recursiveRewrite(value, context, [...parents, key]) + : value; + + return { + ...memo, + [newKey]: newValue, + }; }, {}); }; -const validateFieldValue = (rtType: rt.Any, aggObject: unknown) => { - pipe( - rtType.decode(aggObject), - fold(throwErrors(SavedObjectsErrorHelpers.createBadRequestError), identity) - ); +const lastParent = (parents: string[]) => { + if (parents.length) { + return parents[parents.length - 1]; + } + return undefined; +}; + +const isAttributeKey = (parents: string[]) => { + const last = lastParent(parents); + if (last) { + return attributeMaps.includes(last); + } + return false; +}; + +// hasFilterKeyError(key, types, mappings) +const isAttributeValue = (fieldName: string, fieldValue: unknown): boolean => { + return attributeFields.includes(fieldName) && typeof fieldValue === 'string'; }; + +const validateAndRewriteValue = ( + fieldName: string, + fieldValue: any, + { allowedTypes, indexMapping }: ValidationContext +) => { + const error = hasFilterKeyError(fieldValue, allowedTypes, indexMapping); + if (error) { + throw new Error(error); // TODO: encapsulate + } + return stripAttributesPath(fieldValue); +}; + +const validateAndRewriteKey = ( + fieldName: string, + { allowedTypes, indexMapping }: ValidationContext +) => { + const error = hasFilterKeyError(fieldName, allowedTypes, indexMapping); + if (error) { + throw new Error(error); // TODO: encapsulate + } + return stripAttributesPath(fieldName); +}; + +const stripAttributesPath = (fieldName: string) => fieldName.replace('.attributes', ''); From bfb812d8fc6a09f71ff8d1b9f84c3af53c651bcc Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 11:56:55 +0200 Subject: [PATCH 33/52] some cleanup --- .../service/lib/aggregations/validation.ts | 9 +-------- .../server/saved_objects/service/lib/repository.ts | 10 +++++----- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index e651909f0370b4..107cd4dc356b30 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -18,11 +18,10 @@ const aggregationKeys = ['aggs', 'aggregations']; export const validateAndConvertAggregations = ( allowedTypes: string[], - aggs: Record, + aggs: Record, indexMapping: IndexMapping ): Record => { return validateAggregations(aggs as any, { - // TODO: fix type allowedTypes, indexMapping, currentPath: [], @@ -47,8 +46,6 @@ const validateAggregations = ( aggregations: Record, context: ValidationContext ) => { - // console.log('validateAggregations', Object.keys(aggregations)); - return Object.entries(aggregations).reduce((memo, [aggrName, aggrContainer]) => { return { ...memo, @@ -115,8 +112,6 @@ const validateAggregationType = ( throw new Error(`${aggregationType} aggregation is not valid (or not registered yet)`); } - // console.log('*** validateAggregationType', aggregationType, aggregation); - validateAggregationStructure(aggregationSchema, aggregation, context); return validateAndRewriteFieldAttributes(aggregation, context); }; @@ -132,7 +127,6 @@ const validateAggregationStructure = ( return schema.validate(aggObject, {}, context.currentPath.join('.')); }; -///// /** * List of fields that have an attribute path as value * @@ -203,7 +197,6 @@ const isAttributeKey = (parents: string[]) => { return false; }; -// hasFilterKeyError(key, types, mappings) const isAttributeValue = (fieldName: string, fieldValue: unknown): boolean => { return attributeFields.includes(fieldName) && typeof fieldValue === 'string'; }; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index d867f931a1b134..c0e2cdc3333633 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -820,19 +820,19 @@ export class SavedObjectsRepository { kueryNode = validateConvertFilterToKueryNode(allowedTypes, filter, this._mappings); } catch (e) { if (e.name === 'KQLSyntaxError') { - throw SavedObjectsErrorHelpers.createBadRequestError('KQLSyntaxError: ' + e.message); + throw SavedObjectsErrorHelpers.createBadRequestError(`KQLSyntaxError: ${e.message}`); } else { throw e; } } } - let aggsObject = null; + let aggsObject; if (aggs) { try { aggsObject = validateAndConvertAggregations(allowedTypes, aggs, this._mappings); } catch (e) { - throw e; + throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid aggregation: ${e.message}`); } } @@ -850,7 +850,7 @@ export class SavedObjectsRepository { seq_no_primary_term: true, from: perPage * (page - 1), _source: includedFields(type, fields), - ...(aggsObject != null ? { aggs: aggsObject } : {}), + ...(aggsObject ? { aggs: aggsObject } : {}), ...getSearchDsl(this._mappings, this._registry, { search, defaultSearchOperator, @@ -885,7 +885,7 @@ export class SavedObjectsRepository { } return { - ...(body.aggregations != null ? { aggregations: body.aggregations as any } : {}), + ...(body.aggregations ? { aggregations: (body.aggregations as unknown) as A } : {}), page, per_page: perPage, total: body.hits.total, From 62d3d69fda35ccf08f83e8bad6cf3f0c28536d56 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 13:35:52 +0200 Subject: [PATCH 34/52] minor cleanup --- .../public/saved_objects/saved_objects_client.ts | 1 + src/core/server/saved_objects/types.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 5e28b264660848..782ffa68970484 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -30,6 +30,7 @@ type SavedObjectsFindOptions = Omit< type SavedObjectsFindResponse = Omit>, 'pit_id'>; +/** @public */ export interface SavedObjectsCreateOptions { /** * (Not recommended) Specify an id instead of having the saved objects service generate one for you. diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 08ac6cbc15b585..9ae8793b9ca0a5 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -117,11 +117,17 @@ export interface SavedObjectsFindOptions { defaultSearchOperator?: 'AND' | 'OR'; filter?: string | KueryNode; /** - * Specify an Elasticsearch aggregation to perform. This alpha API only supports a limited set of aggregation types: metrics, bucket. Additional aggregation types can be contributed to Core. - * @alpha + * Specify an Elasticsearch aggregation to perform. + * The API currently only supports a limited set of metrics and bucket aggregation types. + * Additional aggregation types can be contributed to Core. + * * @example - * aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; - * SavedObjects.find({ type: 'dashboard', aggs }) + * ```ts + * const aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; + * return client.find({ type: 'dashboard', aggs }) + * ``` + * + * @alpha */ aggs?: Record; namespaces?: string[]; From 4e2ae77f7d361a53a7406b2bbddf2a77e47c670b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 13:38:45 +0200 Subject: [PATCH 35/52] update generated doc --- .../kibana-plugin-core-public.savedobjectscreateoptions.md | 1 + src/core/public/public.api.md | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md index 086811e2e74e00..a039b9f5b4fe45 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectscreateoptions.md @@ -4,6 +4,7 @@ ## SavedObjectsCreateOptions interface + Signature: ```typescript diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index ddb15e201cd718..006db9cdbba7f1 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1218,8 +1218,6 @@ export class SavedObjectsClient { // @public export type SavedObjectsClientContract = PublicMethodsOf; -// Warning: (ae-missing-release-tag) "SavedObjectsCreateOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export interface SavedObjectsCreateOptions { coreMigrationVersion?: string; From 4bae5b94305b849dda1fd25535778060b4be7fa9 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 13:40:01 +0200 Subject: [PATCH 36/52] adapt telemetry client --- .../server/telemetry_saved_objects_client.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts b/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts index d639b053565d1f..01d89c57311588 100644 --- a/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts +++ b/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts @@ -17,7 +17,9 @@ export class TelemetrySavedObjectsClient extends SavedObjectsClient { * Find the SavedObjects matching the search query in all the Spaces by default * @param options */ - async find(options: SavedObjectsFindOptions): Promise> { + async find( + options: SavedObjectsFindOptions + ): Promise> { return super.find({ namespaces: ['*'], ...options }); } } From 2279447e56812a33e84dce910a84a3f13e606167 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 13:53:00 +0200 Subject: [PATCH 37/52] fix API integ tests --- test/api_integration/apis/saved_objects/find.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index 6ff58ad019dae8..27681655523eaa 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -293,7 +293,7 @@ export default function ({ getService }: FtrProviderContext) { })); }); - describe('with a aggs', () => { + describe('TOTO with a aggs', () => { it('should return 200 with a valid response', async () => await supertest .get( @@ -331,7 +331,7 @@ export default function ({ getService }: FtrProviderContext) { .then((resp) => { expect(resp.body).to.eql({ error: 'Bad Request', - message: 'This type dashboard is not allowed: Bad Request', + message: 'Invalid aggregation: This type dashboard is not allowed: Bad Request', statusCode: 400, }); })); @@ -344,7 +344,7 @@ export default function ({ getService }: FtrProviderContext) { type_count: { max: { field: 'visualization.attributes.version', - script: 'Oh yes I am going to a script', + script: 'Bad script is bad', }, }, }) @@ -355,7 +355,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.eql({ error: 'Bad Request', message: - 'script attribute is not supported in saved objects aggregation: Bad Request', + 'Invalid aggregation: [type_count.max.script]: definition for this key is missing: Bad Request', statusCode: 400, }); })); From 8099f3611311162e0891c8d1d67214299075d2ff Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 14:01:36 +0200 Subject: [PATCH 38/52] fix doc --- docs/api/saved-objects/find.asciidoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/api/saved-objects/find.asciidoc b/docs/api/saved-objects/find.asciidoc index 73a8fbf0a820f7..556048a19e72a2 100644 --- a/docs/api/saved-objects/find.asciidoc +++ b/docs/api/saved-objects/find.asciidoc @@ -53,13 +53,14 @@ experimental[] Retrieve a paginated set of {kib} saved objects by various condit (Optional, object) Filters to objects that have a relationship with the type and ID combination. `filter`:: - (Optional, string) The filter is a KQL string with the caveat that if you filter with an attribute from your type saved object. - It should look like that savedObjectType.attributes.title: "myTitle". However, If you used a direct attribute of a saved object like `updated_at`, - you will have to define your filter like that savedObjectType.updatedAt > 2018-12-22. + (Optional, string) The filter is a KQL string with the caveat that if you filter with an attribute from your saved object type, + it should look like that: `savedObjectType.attributes.title: "myTitle"`. However, If you use a root attribute of a saved + object such as `updated_at`, you will have to define your filter like that: `savedObjectType.updated_at > 2018-12-22`. `aggs`:: - (Optional, string) The aggs will support aggregation string with the caveat that your field from the aggregation will have the attribute from your type saved object, - it should look like this: savedObjectType.attributes.field. However, If you use a direct attribute of a saved object like updatedAt, you will have to define your filter like this: savedObjectType.updated_at. + (Optional, string) **experimental** An aggregation structure, serialized as a string. The field format is similar to `filter`, meaning + that to use a saved object type attribute in the aggregation, the `savedObjectType.attributes.title`: "myTitle"` format + must be used. NOTE: As objects change in {kib}, the results on each page of the response also change. Use the find API for traditional paginated results, but avoid using it to export large amounts of data. From ecc8d2dcf55378a2b59442b1618f3720143324a8 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 14:03:29 +0200 Subject: [PATCH 39/52] TOTO-less --- test/api_integration/apis/saved_objects/find.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index 27681655523eaa..d9ffb63662abdf 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -293,7 +293,7 @@ export default function ({ getService }: FtrProviderContext) { })); }); - describe('TOTO with a aggs', () => { + describe('with a aggs', () => { it('should return 200 with a valid response', async () => await supertest .get( From 08ce7d5b9d694f56c2906d6ee350fe64e0346cc7 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 14:06:44 +0200 Subject: [PATCH 40/52] remove xpack tests --- .../saved_objects/spaces/data.json | 3 +- .../saved_objects/spaces/mappings.json | 231 ------------------ .../saved_object_test_plugin/server/plugin.ts | 9 +- .../common/suites/find.ts | 74 ++---- .../security_and_spaces/apis/find.ts | 4 - .../security_only/apis/find.ts | 4 - 6 files changed, 16 insertions(+), 309 deletions(-) diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json index 7b8e5961126a6a..5fac012d5e8b96 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json @@ -338,8 +338,7 @@ "index": ".kibana", "source": { "globaltype": { - "title": "My favorite global object", - "version": 1 + "title": "My favorite global object" }, "type": "globaltype", "updated_at": "2017-09-21T18:59:16.270Z" diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index 59569be17e8563..12606562ab33b5 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -9,237 +9,6 @@ "mappings": { "dynamic": "false", "properties": { - "config": { - "dynamic": "true", - "properties": { - "buildNum": { - "type": "keyword" - }, - "defaultIndex": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "dashboard": { - "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "optionsJSON": { - "type": "text" - }, - "panelsJSON": { - "type": "text" - }, - "refreshInterval": { - "properties": { - "display": { - "type": "keyword" - }, - "pause": { - "type": "boolean" - }, - "section": { - "type": "integer" - }, - "value": { - "type": "integer" - } - } - }, - "timeFrom": { - "type": "keyword" - }, - "timeRestore": { - "type": "boolean" - }, - "timeTo": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "globaltype": { - "properties": { - "title": { - "fields": { - "keyword": { - "ignore_above": 2048, - "type": "keyword" - } - }, - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "graph-workspace": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "numLinks": { - "type": "integer" - }, - "numVertices": { - "type": "integer" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "wsState": { - "type": "text" - } - } - }, - "index-pattern": { - "properties": { - "fieldFormatMap": { - "type": "text" - }, - "fields": { - "type": "text" - }, - "intervalName": { - "type": "keyword" - }, - "notExpandable": { - "type": "boolean" - }, - "sourceFilters": { - "type": "text" - }, - "timeFieldName": { - "type": "keyword" - }, - "title": { - "type": "text" - } - } - }, - "isolatedtype": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "savedSearchId": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "visState": { - "type": "text" - } - } - }, - "namespace": { - "type": "keyword" - }, - "namespaces": { - "type": "keyword" - }, - "originId": { - "type": "keyword" - }, - "search": { - "properties": { - "columns": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "sort": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "server": { - "properties": { - "uuid": { - "type": "keyword" - } - } - }, - "sharedtype": { - "properties": { - "title": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 2048 - } - } - } - } - }, "space": { "dynamic": false, "type": "object" diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts b/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts index 933ade68863cb9..e29bbc0db56b6a 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts +++ b/x-pack/test/saved_object_api_integration/common/fixtures/saved_object_test_plugin/server/plugin.ts @@ -64,14 +64,7 @@ export class Plugin { hidden: false, namespaceType: 'agnostic', management, - mappings: { - properties: { - ...mappings.properties, - version: { - type: 'integer', - }, - }, - }, + mappings, }); core.savedObjects.registerType({ name: 'hiddentype', diff --git a/x-pack/test/saved_object_api_integration/common/suites/find.ts b/x-pack/test/saved_object_api_integration/common/suites/find.ts index 44ba2d7148d581..27905459c29b77 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/find.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/find.ts @@ -36,10 +36,9 @@ export interface FindTestCase { total?: number; }; failure?: { - statusCode: 200 | 400 | 403; // if the user searches for types and/or namespaces they are not authorized for, they will get a 200 result with those types/namespaces omitted + statusCode: 200 | 400; // if the user searches for types and/or namespaces they are not authorized for, they will get a 200 result with those types/namespaces omitted reason: 'unauthorized' | 'cross_namespace_not_permitted' | 'bad_request'; }; - typeUseInQueryField?: string; } const TEST_CASES = [ @@ -52,7 +51,7 @@ export const getTestCases = ( currentSpace: undefined, crossSpaceSearch: undefined, } -): Record => { +) => { const crossSpaceIds = crossSpaceSearch?.filter((s) => s !== (currentSpace ?? DEFAULT_SPACE_ID)) ?? []; // intentionally exclude the current space const isCrossSpaceSearch = crossSpaceIds.length > 0; @@ -99,7 +98,7 @@ export const getTestCases = ( successResult: { savedObjects: getExpectedSavedObjects((t) => t.type === 'isolatedtype'), }, - }, + } as FindTestCase, multiNamespaceType: { title: buildTitle('find multi-namespace type'), query: `type=sharedtype&fields=title${namespacesQueryParam}`, @@ -119,15 +118,15 @@ export const getTestCases = ( title: buildTitle('find namespace-agnostic type'), query: `type=globaltype&fields=title${namespacesQueryParam}`, successResult: { savedObjects: SAVED_OBJECT_TEST_CASES.NAMESPACE_AGNOSTIC }, - }, + } as FindTestCase, hiddenType: { title: buildTitle('find hidden type'), query: `type=hiddentype&fields=name${namespacesQueryParam}`, - }, + } as FindTestCase, unknownType: { title: buildTitle('find unknown type'), query: `type=wigwags${namespacesQueryParam}`, - }, + } as FindTestCase, eachType: { title: buildTitle('find each type'), query: `type=isolatedtype&type=sharedtype&type=globaltype&type=hiddentype&type=wigwags${namespacesQueryParam}`, @@ -136,7 +135,7 @@ export const getTestCases = ( ['isolatedtype', 'sharedtype', 'globaltype'].includes(t.type) ), }, - }, + } as FindTestCase, pageBeyondTotal: { title: buildTitle('find page beyond total'), query: `type=isolatedtype&page=100&per_page=100${namespacesQueryParam}`, @@ -146,27 +145,24 @@ export const getTestCases = ( total: -1, savedObjects: [], }, - }, + } as FindTestCase, unknownSearchField: { title: buildTitle('find unknown search field'), query: `type=url&search_fields=a${namespacesQueryParam}`, - }, + } as FindTestCase, filterWithNamespaceAgnosticType: { title: buildTitle('filter with namespace-agnostic type'), query: `type=globaltype&filter=globaltype.attributes.title:*global*${namespacesQueryParam}`, successResult: { savedObjects: SAVED_OBJECT_TEST_CASES.NAMESPACE_AGNOSTIC }, - typeUseInQueryField: 'globaltype', - }, + } as FindTestCase, filterWithHiddenType: { title: buildTitle('filter with hidden type'), query: `type=hiddentype&fields=name&filter=hiddentype.attributes.title:'hello'${namespacesQueryParam}`, - typeUseInQueryField: 'hiddentype', - }, + } as FindTestCase, filterWithUnknownType: { title: buildTitle('filter with unknown type'), query: `type=wigwags&filter=wigwags.attributes.title:'unknown'${namespacesQueryParam}`, - typeUseInQueryField: 'wigwags', - }, + } as FindTestCase, filterWithDisallowedType: { title: buildTitle('filter with disallowed type'), query: `type=globaltype&filter=dashboard.title:'Requests'${namespacesQueryParam}`, @@ -174,49 +170,7 @@ export const getTestCases = ( statusCode: 400, reason: 'bad_request', }, - typeUseInQueryField: 'dashboard', - }, - aggsWithNamespaceAgnosticType: { - title: buildTitle('aggs with namespace-agnostic type'), - query: `type=globaltype&aggs=${encodeURIComponent( - JSON.stringify({ - type_count: { max: { field: 'globaltype.attributes.version' } }, - }) - )}${namespacesQueryParam}`, - successResult: { savedObjects: SAVED_OBJECT_TEST_CASES.NAMESPACE_AGNOSTIC }, - typeUseInQueryField: 'globaltype', - }, - aggsWithHiddenType: { - title: buildTitle('aggs with hidden type'), - query: `type=hiddentype&fields=name&aggs=${encodeURIComponent( - JSON.stringify({ - type_count: { max: { field: 'hiddentype.attributes.title' } }, - }) - )}${namespacesQueryParam}`, - typeUseInQueryField: 'hiddentype', - }, - aggsWithUnknownType: { - title: buildTitle('aggs with unknown type'), - query: `type=wigwags&aggs=${encodeURIComponent( - JSON.stringify({ - type_count: { max: { field: 'wigwags.attributes.version' } }, - }) - )}${namespacesQueryParam}`, - typeUseInQueryField: 'wigwags', - }, - aggsWithDisallowedType: { - title: buildTitle('aggs with disallowed type'), - query: `type=globaltype&aggs=${encodeURIComponent( - JSON.stringify({ - type_count: { max: { field: 'dashboard.attributes.version' } }, - }) - )}${namespacesQueryParam}`, - failure: { - statusCode: 400, - reason: 'bad_request', - }, - typeUseInQueryField: 'dashboard', - }, + } as FindTestCase, }; }; @@ -250,7 +204,7 @@ export function findTestSuiteFactory(esArchiver: any, supertest: SuperTest) } } else if (failure?.statusCode === 400) { if (failure.reason === 'bad_request') { - const type = testCase.typeUseInQueryField ?? 'unknown type'; + const type = (parsedQuery.filter as string).split('.')[0]; expect(response.body.error).to.eql('Bad Request'); expect(response.body.statusCode).to.eql(failure.statusCode); expect(response.body.message).to.eql(`This type ${type} is not allowed: Bad Request`); diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts index 0f7a1dc93bf3d7..6d9c38ecca5962 100644 --- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/find.ts @@ -34,16 +34,12 @@ const createTestCases = (currentSpace: string, crossSpaceSearch?: string[]) => { cases.unknownSearchField, cases.filterWithNamespaceAgnosticType, cases.filterWithDisallowedType, - cases.aggsWithNamespaceAgnosticType, - cases.aggsWithDisallowedType, ]; const hiddenAndUnknownTypes = [ cases.hiddenType, cases.unknownType, cases.filterWithHiddenType, cases.filterWithUnknownType, - cases.aggsWithHiddenType, - cases.aggsWithUnknownType, ]; const allTypes = normalTypes.concat(hiddenAndUnknownTypes); return { normalTypes, hiddenAndUnknownTypes, allTypes }; diff --git a/x-pack/test/saved_object_api_integration/security_only/apis/find.ts b/x-pack/test/saved_object_api_integration/security_only/apis/find.ts index ea47c66004982e..eb30024015fbb2 100644 --- a/x-pack/test/saved_object_api_integration/security_only/apis/find.ts +++ b/x-pack/test/saved_object_api_integration/security_only/apis/find.ts @@ -31,16 +31,12 @@ const createTestCases = (crossSpaceSearch?: string[]) => { cases.unknownSearchField, cases.filterWithNamespaceAgnosticType, cases.filterWithDisallowedType, - cases.aggsWithNamespaceAgnosticType, - cases.aggsWithDisallowedType, ]; const hiddenAndUnknownTypes = [ cases.hiddenType, cases.unknownType, cases.filterWithHiddenType, cases.filterWithUnknownType, - cases.aggsWithHiddenType, - cases.aggsWithUnknownType, ]; const allTypes = normalTypes.concat(hiddenAndUnknownTypes); return { normalTypes, hiddenAndUnknownTypes, allTypes }; From 2aef2d8f4218a76dc7b06248567023805604fbf7 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 14:21:12 +0200 Subject: [PATCH 41/52] list supported / unsupported aggregations --- .../aggregations/aggs_types/bucket_aggs.ts | 34 +++++++++ .../aggregations/aggs_types/metrics_aggs.ts | 71 +++++++++++-------- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts index 1c98653aa6720f..1508cab69a0486 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -8,6 +8,40 @@ import { schema as s, ObjectType } from '@kbn/config-schema'; +/** + * Schemas for the Bucket aggregations. + * + * Currently supported: + * - filter + * - histogram + * - terms + * + * Not implemented: + * - adjacency_matrix + * - auto_date_histogram + * - children + * - composite + * - date_histogram + * - date_range + * - diversified_sampler + * - filters + * - geo_distance + * - geohash_grid + * - geotile_grid + * - global + * - ip_range + * - missing + * - multi_terms + * - nested + * - parent + * - range + * - rare_terms + * - reverse_nested + * - sampler + * - significant_terms + * - significant_text + * - variable_width_histogram + */ export const bucketAggsSchemas: Record = { filter: s.object({ term: s.recordOf(s.string(), s.oneOf([s.string(), s.boolean(), s.number()])), diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts index 85dec9614675cd..c05ae67cd2164e 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts @@ -8,24 +8,35 @@ import { schema as s, ObjectType } from '@kbn/config-schema'; -/* - * Types for Metrics Aggregations +/** + * Schemas for the metrics Aggregations + * + * Currently supported: + * - avg + * - cardinality + * - min + * - max + * - sum + * - top_hits + * - weighted_avg * * Not implemented: - * - Extended Stats Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-extendedstats-aggregation.html - * - Geo Bounds Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geobounds-aggregation.html - * - Geo Centroid Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geocentroid-aggregation.html - * - Percentiles Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-percentile-aggregation.html - * - Percentile Ranks Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-percentile-rank-aggregation.html - * - Scripted Metric Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-scripted-metric-aggregation.html - * - Stats Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-stats-aggregation.html - * - String Stats Aggregation (x-pack) https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-string-stats-aggregation.html - * - Sum Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-sum-aggregation.html - * - Top Hits Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-top-hits-aggregation.html - * - Value Count Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-valuecount-aggregation.html - * - Median Absolute Deviation Aggregation https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-median-absolute-deviation-aggregation.html + * - boxplot + * - extended_stats + * - geo_bounds + * - geo_centroid + * - geo_line + * - matrix_stats + * - median_absolute_deviation + * - percentile_ranks + * - percentiles + * - rate + * - scripted_metric + * - stats + * - string_stats + * - t_test + * - value_count */ - export const metricsAggsSchemas: Record = { avg: s.object({ field: s.maybe(s.string()), @@ -47,6 +58,23 @@ export const metricsAggsSchemas: Record = { missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), format: s.maybe(s.string()), }), + sum: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + top_hits: s.object({ + explain: s.maybe(s.boolean()), + docvalue_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + from: s.maybe(s.number()), + size: s.maybe(s.number()), + sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + seq_no_primary_term: s.maybe(s.boolean()), + version: s.maybe(s.boolean()), + track_scores: s.maybe(s.boolean()), + highlight: s.maybe(s.any()), + _source: s.maybe(s.oneOf([s.boolean(), s.string(), s.arrayOf(s.string())])), + }), weighted_avg: s.object({ format: s.maybe(s.string()), value_type: s.maybe(s.string()), @@ -63,17 +91,4 @@ export const metricsAggsSchemas: Record = { }) ), }), - top_hits: s.object({ - explain: s.maybe(s.boolean()), - docvalue_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), - stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), - from: s.maybe(s.number()), - size: s.maybe(s.number()), - sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), - seq_no_primary_term: s.maybe(s.boolean()), - version: s.maybe(s.boolean()), - track_scores: s.maybe(s.boolean()), - highlight: s.maybe(s.any()), - _source: s.maybe(s.oneOf([s.boolean(), s.string(), s.arrayOf(s.string())])), - }), }; From 0ffe8049832ca0a795c1b4ff0ff60fb0a973e90f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 14:23:42 +0200 Subject: [PATCH 42/52] typo fix --- src/core/server/saved_objects/service/lib/filter_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index 0c947a96466f39..b3bcef9a62e130 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -234,7 +234,7 @@ export const fieldDefined = (indexMappings: IndexMapping, key: string): boolean return true; } - // If the path is for a flattned type field, we'll assume the mappings are defined. + // If the path is for a flattened type field, we'll assume the mappings are defined. const keys = key.split('.'); for (let i = 0; i < keys.length; i++) { const path = `properties.${keys.slice(0, i + 1).join('.properties.')}`; From 90dc2a102136f6dc47d39c376b8bf19f9c95aa3e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 21:12:46 +0200 Subject: [PATCH 43/52] extract some validation function --- .../lib/aggregations/validation.test.ts | 12 +- .../service/lib/aggregations/validation.ts | 84 +++++----- .../lib/aggregations/validation_utils.test.ts | 148 ++++++++++++++++++ .../lib/aggregations/validation_utils.ts | 80 ++++++++++ 4 files changed, 279 insertions(+), 45 deletions(-) create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index 4bad2d92140fe9..9d077a6e9fd88a 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -231,7 +231,7 @@ describe('Aggregation Utils', () => { expect(() => validateAndConvertAggregations(['alert'], aggregations, mockMappings) ).toThrowErrorMatchingInlineSnapshot( - `"This key 'alert.actions.group' does NOT match the filter proposition SavedObjectType.attributes.key"` + `"[average.avg.field] Invalid attribute path: alert.actions.group"` ); }); @@ -247,7 +247,7 @@ describe('Aggregation Utils', () => { expect(() => validateAndConvertAggregations(['alert'], aggregations, mockMappings) ).toThrowErrorMatchingInlineSnapshot( - `"This key 'alert.attributes.actions.non_existing' does NOT exist in alert saved object index patterns"` + `"[average.avg.field] Invalid attribute path: alert.attributes.actions.non_existing"` ); }); @@ -311,7 +311,9 @@ describe('Aggregation Utils', () => { expect(() => { validateAndConvertAggregations(['alert'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot(`"This type foo is not allowed"`); + }).toThrowErrorMatchingInlineSnapshot( + `"[myFilter.max.field] Invalid attribute path: foo.attributes.bytes"` + ); }); it('throws an error when an attributes is not respecting its schema definition', () => { @@ -342,7 +344,7 @@ describe('Aggregation Utils', () => { expect(() => { validateAndConvertAggregations(['foo'], aggregations, mockMappings); }).toThrowErrorMatchingInlineSnapshot( - `"auto_date_histogram aggregation is not valid (or not registered yet)"` + `"[someAgg.auto_date_histogram] auto_date_histogram aggregation is not valid (or not registered yet)"` ); }); @@ -365,7 +367,7 @@ describe('Aggregation Utils', () => { expect(() => { validateAndConvertAggregations(['foo'], aggregations, mockMappings); }).toThrowErrorMatchingInlineSnapshot( - `"cumulative_cardinality aggregation is not valid (or not registered yet)"` + `"[someAgg.aggs.unknownAgg.cumulative_cardinality] cumulative_cardinality aggregation is not valid (or not registered yet)"` ); }); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index 107cd4dc356b30..899061c7797a17 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -11,29 +11,40 @@ import { ObjectType } from '@kbn/config-schema'; import { isPlainObject } from 'lodash'; import { IndexMapping } from '../../../mappings'; -import { hasFilterKeyError } from '../filter_utils'; +import { + isObjectTypeAttribute, + rewriteObjectTypeAttribute, + isRootLevelAttribute, + rewriteRootLevelAttribute, +} from './validation_utils'; import { aggregationSchemas } from './aggs_types'; const aggregationKeys = ['aggs', 'aggregations']; +interface ValidationContext { + allowedTypes: string[]; + indexMapping: IndexMapping; + currentPath: string[]; +} + +/** + * Validates an aggregation structure against the declared mapping and + * aggregation schema, and rewrite the attribute fields from the + * `{type}.attributes.{attribute}` (KQL-like) syntax to the underlying + * `{type}.{attribute} format.` + */ export const validateAndConvertAggregations = ( allowedTypes: string[], aggs: Record, indexMapping: IndexMapping ): Record => { - return validateAggregations(aggs as any, { + return validateAggregations(aggs, { allowedTypes, indexMapping, currentPath: [], }); }; -interface ValidationContext { - allowedTypes: string[]; - indexMapping: IndexMapping; - currentPath: string[]; -} - /** * Validates a record of aggregation containers, * Which can either be the root level aggregations (`SearchRequest.body.aggs`) @@ -51,14 +62,7 @@ const validateAggregations = ( ...memo, [aggrName]: validateAggregation(aggrContainer, childContext(context, aggrName)), }; - }, {}); -}; - -const childContext = (context: ValidationContext, path: string): ValidationContext => { - return { - ...context, - currentPath: [...context.currentPath, path], - }; + }, {} as Record); }; /** @@ -109,7 +113,11 @@ const validateAggregationType = ( ) => { const aggregationSchema = aggregationSchemas[aggregationType]; if (!aggregationSchema) { - throw new Error(`${aggregationType} aggregation is not valid (or not registered yet)`); + throw new Error( + `[${context.currentPath.join( + '.' + )}] ${aggregationType} aggregation is not valid (or not registered yet)` + ); } validateAggregationStructure(aggregationSchema, aggregation, context); @@ -168,11 +176,12 @@ const recursiveRewrite = ( const rewriteKey = isAttributeKey(parents); const rewriteValue = isAttributeValue(key, value); - const newKey = rewriteKey ? validateAndRewriteKey(key, context) : key; + const nestedContext = childContext(context, key); + const newKey = rewriteKey ? validateAndRewriteAttributePath(key, nestedContext) : key; const newValue = rewriteValue - ? validateAndRewriteValue(key, value, context) + ? validateAndRewriteAttributePath(value, nestedContext) : isPlainObject(value) - ? recursiveRewrite(value, context, [...parents, key]) + ? recursiveRewrite(value, nestedContext, [...parents, key]) : value; return { @@ -182,6 +191,13 @@ const recursiveRewrite = ( }, {}); }; +const childContext = (context: ValidationContext, path: string): ValidationContext => { + return { + ...context, + currentPath: [...context.currentPath, path], + }; +}; + const lastParent = (parents: string[]) => { if (parents.length) { return parents[parents.length - 1]; @@ -201,27 +217,15 @@ const isAttributeValue = (fieldName: string, fieldValue: unknown): boolean => { return attributeFields.includes(fieldName) && typeof fieldValue === 'string'; }; -const validateAndRewriteValue = ( - fieldName: string, - fieldValue: any, - { allowedTypes, indexMapping }: ValidationContext +const validateAndRewriteAttributePath = ( + attributePath: string, + { allowedTypes, indexMapping, currentPath }: ValidationContext ) => { - const error = hasFilterKeyError(fieldValue, allowedTypes, indexMapping); - if (error) { - throw new Error(error); // TODO: encapsulate + if (isRootLevelAttribute(attributePath, indexMapping, allowedTypes)) { + return rewriteRootLevelAttribute(attributePath); } - return stripAttributesPath(fieldValue); -}; - -const validateAndRewriteKey = ( - fieldName: string, - { allowedTypes, indexMapping }: ValidationContext -) => { - const error = hasFilterKeyError(fieldName, allowedTypes, indexMapping); - if (error) { - throw new Error(error); // TODO: encapsulate + if (isObjectTypeAttribute(attributePath, indexMapping, allowedTypes)) { + return rewriteObjectTypeAttribute(attributePath); } - return stripAttributesPath(fieldName); + throw new Error(`[${currentPath.join('.')}] Invalid attribute path: ${attributePath}`); }; - -const stripAttributesPath = (fieldName: string) => fieldName.replace('.attributes', ''); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts new file mode 100644 index 00000000000000..25c3aea474ecec --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndexMapping } from '../../../mappings'; +import { + isRootLevelAttribute, + rewriteRootLevelAttribute, + isObjectTypeAttribute, + rewriteObjectTypeAttribute, +} from './validation_utils'; + +const mockMappings: IndexMapping = { + properties: { + updated_at: { + type: 'date', + }, + foo: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + bytes: { + type: 'number', + }, + }, + }, + bean: { + properties: { + canned: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + alert: { + properties: { + actions: { + type: 'nested', + properties: { + group: { + type: 'keyword', + }, + actionRef: { + type: 'keyword', + }, + actionTypeId: { + type: 'keyword', + }, + params: { + enabled: false, + type: 'object', + }, + }, + }, + params: { + type: 'flattened', + }, + }, + }, + }, +}; + +describe('isRootLevelAttribute', () => { + it('returns true when referring to a path to a valid root level field', () => { + expect(isRootLevelAttribute('foo.updated_at', mockMappings, ['foo'])).toBe(true); + }); + it('returns false when referring to a direct path to a valid root level field', () => { + expect(isRootLevelAttribute('updated_at', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a unknown root level field', () => { + expect(isRootLevelAttribute('foo.not_present', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to an existing nested field', () => { + expect(isRootLevelAttribute('foo.properties.title', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a valid root level field of an unknown type', () => { + expect(isRootLevelAttribute('bar.updated_at', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a valid root level type field', () => { + expect(isRootLevelAttribute('foo.foo', mockMappings, ['foo'])).toBe(false); + }); +}); + +describe('rewriteRootLevelAttribute', () => { + it('rewrites the attribute path to strip the type', () => { + expect(rewriteRootLevelAttribute('foo.references')).toEqual('references'); + }); + it('does not handle real root level path', () => { + expect(rewriteRootLevelAttribute('references')).not.toEqual('references'); + }); +}); + +describe('isObjectTypeAttribute', () => { + it('return true if attribute path is valid', () => { + expect(isObjectTypeAttribute('foo.attributes.description', mockMappings, ['foo'])).toEqual( + true + ); + }); + + it('return true for nested attributes', () => { + expect(isObjectTypeAttribute('bean.attributes.canned.text', mockMappings, ['bean'])).toEqual( + true + ); + }); + + it('return false if attribute path points to an invalid type', () => { + expect(isObjectTypeAttribute('foo.attributes.description', mockMappings, ['bean'])).toEqual( + false + ); + }); + + it('returns false if attribute path refers to a type', () => { + expect(isObjectTypeAttribute('bean', mockMappings, ['bean'])).toEqual(false); + }); + + it('Return error if key does not match SO attribute structure', () => { + expect(isObjectTypeAttribute('bean.canned.text', mockMappings, ['bean'])).toEqual(false); + }); + + it('Return false if key matches nested type attribute parent', () => { + expect(isObjectTypeAttribute('alert.actions', mockMappings, ['alert'])).toEqual(false); + }); + + it('returns false if path refers to a non-existent attribute', () => { + expect(isObjectTypeAttribute('bean.attributes.red', mockMappings, ['bean'])).toEqual(false); + }); +}); + +describe('rewriteObjectTypeAttribute', () => { + it('rewrites the attribute path to strip the type', () => { + expect(rewriteObjectTypeAttribute('foo.attributes.prop')).toEqual('foo.prop'); + }); + it('returns invalid input unchanged', () => { + expect(rewriteObjectTypeAttribute('foo.references')).toEqual('foo.references'); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts new file mode 100644 index 00000000000000..f817497e3759e9 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndexMapping } from '../../../mappings'; +import { fieldDefined, hasFilterKeyError } from '../filter_utils'; + +/** + * Returns true if the given attribute path is a valid root level SO attribute path + * + * @example + * ```ts + * isRootLevelAttribute('myType.updated_at', indexMapping, ['myType']}) + * // => true + * ``` + */ +export const isRootLevelAttribute = ( + attributePath: string, + indexMapping: IndexMapping, + allowedTypes: string[] +): boolean => { + const splits = attributePath.split('.'); + if (splits.length !== 2) { + return false; + } + + const [type, fieldName] = splits; + if (allowedTypes.includes(fieldName)) { + return false; + } + return allowedTypes.includes(type) && fieldDefined(indexMapping, fieldName); +}; + +/** + * Rewrites a root level attribute path to strip the type + * + * @example + * ```ts + * rewriteRootLevelAttribute('myType.updated_at') + * // => 'updated_at' + * ``` + */ +export const rewriteRootLevelAttribute = (attributePath: string) => { + return attributePath.split('.')[1]; +}; + +/** + * Returns true if the given attribute path is a valid object type level SO attribute path + * + * @example + * ```ts + * isObjectTypeAttribute('myType.attributes.someField', indexMapping, ['myType']}) + * // => true + * ``` + */ +export const isObjectTypeAttribute = ( + attributePath: string, + indexMapping: IndexMapping, + allowedTypes: string[] +): boolean => { + const error = hasFilterKeyError(attributePath, allowedTypes, indexMapping); + return error == null; +}; + +/** + * Rewrites a object type attribute path to strip the type + * + * @example + * ```ts + * rewriteObjectTypeAttribute('myType.attributes.foo') + * // => 'myType.foo' + * ``` + */ +export const rewriteObjectTypeAttribute = (attributePath: string) => { + return attributePath.replace('.attributes', ''); +}; From ccc50df66b9a99ba031d2da0e3baa9cbcd5ff9f5 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 21:16:07 +0200 Subject: [PATCH 44/52] fix indent --- .../lib/aggregations/validation.test.ts | 531 +++++++++--------- 1 file changed, 256 insertions(+), 275 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index 9d077a6e9fd88a..6a1d2cef532ff5 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -29,16 +29,6 @@ const mockMappings = { }, }, }, - bar: { - properties: { - foo: { - type: 'text', - }, - description: { - type: 'text', - }, - }, - }, bean: { properties: { canned: { @@ -76,340 +66,331 @@ const mockMappings = { }, }, }, - hiddenType: { - properties: { - description: { - type: 'text', - }, - }, - }, }, }; -describe('Aggregation Utils', () => { - describe('#validateGetSavedObjectsAggs', () => { - it('Validates a simple aggregations', () => { - expect( - validateAndConvertAggregations( - ['foo'], - { aggName: { max: { field: 'foo.attributes.bytes' } } }, - mockMappings - ) - ).toEqual({ - aggName: { - max: { - field: 'foo.bytes', - }, +describe('validateAndConvertAggregations', () => { + it('Validates a simple aggregations', () => { + expect( + validateAndConvertAggregations( + ['foo'], + { aggName: { max: { field: 'foo.attributes.bytes' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + max: { + field: 'foo.bytes', }, - }); + }, }); + }); - it('validates a nested field in simple aggregations', () => { - expect( - validateAndConvertAggregations( - ['alert'], - { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, - mockMappings - ) - ).toEqual({ - aggName: { - cardinality: { - field: 'alert.actions.group', - }, + it('validates a nested field in simple aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', }, - }); + }, }); + }); - it('validates a nested aggregations', () => { - expect( - validateAndConvertAggregations( - ['alert'], - { - aggName: { - cardinality: { - field: 'alert.attributes.actions.group', - }, - aggs: { - aggName: { - max: { field: 'alert.attributes.actions.group' }, - }, + it('validates a nested aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { + aggName: { + cardinality: { + field: 'alert.attributes.actions.group', + }, + aggs: { + aggName: { + max: { field: 'alert.attributes.actions.group' }, }, }, }, - mockMappings - ) - ).toEqual({ - aggName: { - cardinality: { - field: 'alert.actions.group', - }, - aggs: { - aggName: { - max: { - field: 'alert.actions.group', - }, + }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', + }, + aggs: { + aggName: { + max: { + field: 'alert.actions.group', }, }, }, - }); + }, }); + }); - it('validates a deeply nested aggregations', () => { - expect( - validateAndConvertAggregations( - ['alert'], - { - first: { - cardinality: { - field: 'alert.attributes.actions.group', - }, - aggs: { - second: { - max: { field: 'alert.attributes.actions.group' }, - aggs: { - third: { - min: { - field: 'alert.attributes.actions.actionTypeId', - }, + it('validates a deeply nested aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { + first: { + cardinality: { + field: 'alert.attributes.actions.group', + }, + aggs: { + second: { + max: { field: 'alert.attributes.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.attributes.actions.actionTypeId', }, }, }, }, }, }, - mockMappings - ) - ).toEqual({ - first: { - cardinality: { - field: 'alert.actions.group', - }, - aggs: { - second: { - max: { field: 'alert.actions.group' }, - aggs: { - third: { - min: { - field: 'alert.actions.actionTypeId', - }, + }, + mockMappings + ) + ).toEqual({ + first: { + cardinality: { + field: 'alert.actions.group', + }, + aggs: { + second: { + max: { field: 'alert.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.actions.actionTypeId', }, }, }, }, }, - }); + }, }); + }); - it('rewrites the `field` name when valid', () => { - const aggregations: AggsMap = { - average: { - avg: { - field: 'alert.attributes.actions.group', - missing: 10, - }, + it('rewrites the `field` name when valid', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.group', + missing: 10, }, - }; - expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ - average: { - avg: { - field: 'alert.actions.group', - missing: 10, - }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + avg: { + field: 'alert.actions.group', + missing: 10, }, - }); + }, }); + }); - it('throws an error when the `field` name is not using attributes path', () => { - const aggregations: AggsMap = { - average: { - avg: { - field: 'alert.actions.group', - missing: 10, - }, + it('throws an error when the `field` name is not using attributes path', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.actions.group', + missing: 10, }, - }; - expect(() => - validateAndConvertAggregations(['alert'], aggregations, mockMappings) - ).toThrowErrorMatchingInlineSnapshot( - `"[average.avg.field] Invalid attribute path: alert.actions.group"` - ); - }); + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.actions.group"` + ); + }); - it('throws an error when the `field` name is referencing an invalid field', () => { - const aggregations: AggsMap = { - average: { - avg: { - field: 'alert.attributes.actions.non_existing', - missing: 10, - }, + it('throws an error when the `field` name is referencing an invalid field', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.non_existing', + missing: 10, }, - }; - expect(() => - validateAndConvertAggregations(['alert'], aggregations, mockMappings) - ).toThrowErrorMatchingInlineSnapshot( - `"[average.avg.field] Invalid attribute path: alert.attributes.actions.non_existing"` - ); - }); + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.attributes.actions.non_existing"` + ); + }); - it('rewrites the `field` name even when nested', () => { - const aggregations: AggsMap = { - average: { - weighted_avg: { - value: { - field: 'alert.attributes.actions.group', - missing: 10, - }, - weight: { - field: 'alert.attributes.actions.actionRef', - }, + it('rewrites the `field` name even when nested', () => { + const aggregations: AggsMap = { + average: { + weighted_avg: { + value: { + field: 'alert.attributes.actions.group', + missing: 10, + }, + weight: { + field: 'alert.attributes.actions.actionRef', }, }, - }; - expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ - average: { - weighted_avg: { - value: { - field: 'alert.actions.group', - missing: 10, - }, - weight: { - field: 'alert.actions.actionRef', - }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + weighted_avg: { + value: { + field: 'alert.actions.group', + missing: 10, + }, + weight: { + field: 'alert.actions.actionRef', }, }, - }); + }, }); + }); - it('rewrites the entries of a filter term record', () => { - const aggregations: AggsMap = { - myFilter: { - filter: { - term: { - 'foo.attributes.description': 'hello', - 'foo.attributes.bytes': 10, - }, + it('rewrites the entries of a filter term record', () => { + const aggregations: AggsMap = { + myFilter: { + filter: { + term: { + 'foo.attributes.description': 'hello', + 'foo.attributes.bytes': 10, }, }, - }; - expect(validateAndConvertAggregations(['foo'], aggregations, mockMappings)).toEqual({ - myFilter: { - filter: { - term: { 'foo.description': 'hello', 'foo.bytes': 10 }, - }, + }, + }; + expect(validateAndConvertAggregations(['foo'], aggregations, mockMappings)).toEqual({ + myFilter: { + filter: { + term: { 'foo.description': 'hello', 'foo.bytes': 10 }, }, - }); + }, }); + }); - it('throws an error when referencing non-allowed types', () => { - const aggregations: AggsMap = { - myFilter: { - max: { - field: 'foo.attributes.bytes', - }, + it('throws an error when referencing non-allowed types', () => { + const aggregations: AggsMap = { + myFilter: { + max: { + field: 'foo.attributes.bytes', }, - }; + }, + }; - expect(() => { - validateAndConvertAggregations(['alert'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot( - `"[myFilter.max.field] Invalid attribute path: foo.attributes.bytes"` - ); - }); + expect(() => { + validateAndConvertAggregations(['alert'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[myFilter.max.field] Invalid attribute path: foo.attributes.bytes"` + ); + }); - it('throws an error when an attributes is not respecting its schema definition', () => { - const aggregations: AggsMap = { - someAgg: { - terms: { - missing: 'expecting a number', - }, + it('throws an error when an attributes is not respecting its schema definition', () => { + const aggregations: AggsMap = { + someAgg: { + terms: { + missing: 'expecting a number', }, - }; + }, + }; - expect(() => - validateAndConvertAggregations(['alert'], aggregations, mockMappings) - ).toThrowErrorMatchingInlineSnapshot( - `"[someAgg.terms.missing]: expected value of type [number] but got [string]"` - ); - }); + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.terms.missing]: expected value of type [number] but got [string]"` + ); + }); - it('throws an error when trying to validate an unknown aggregation type', () => { - const aggregations: AggsMap = { - someAgg: { - auto_date_histogram: { - field: 'foo.attributes.bytes', - }, + it('throws an error when trying to validate an unknown aggregation type', () => { + const aggregations: AggsMap = { + someAgg: { + auto_date_histogram: { + field: 'foo.attributes.bytes', }, - }; + }, + }; - expect(() => { - validateAndConvertAggregations(['foo'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot( - `"[someAgg.auto_date_histogram] auto_date_histogram aggregation is not valid (or not registered yet)"` - ); - }); + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.auto_date_histogram] auto_date_histogram aggregation is not valid (or not registered yet)"` + ); + }); - it('throws an error when a child aggregation is unknown', () => { - const aggregations: AggsMap = { - someAgg: { - max: { - field: 'foo.attributes.bytes', - }, - aggs: { - unknownAgg: { - cumulative_cardinality: { - format: 'format', - }, + it('throws an error when a child aggregation is unknown', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + }, + aggs: { + unknownAgg: { + cumulative_cardinality: { + format: 'format', }, }, }, - }; + }, + }; - expect(() => { - validateAndConvertAggregations(['foo'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot( - `"[someAgg.aggs.unknownAgg.cumulative_cardinality] cumulative_cardinality aggregation is not valid (or not registered yet)"` - ); - }); + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.aggs.unknownAgg.cumulative_cardinality] cumulative_cardinality aggregation is not valid (or not registered yet)"` + ); + }); - it('throws an error when using a script attribute', () => { - const aggregations: AggsMap = { - someAgg: { - max: { - field: 'foo.attributes.bytes', - script: 'This is a bad script', - }, + it('throws an error when using a script attribute', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', }, - }; + }, + }; - expect(() => { - validateAndConvertAggregations(['foo'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot( - `"[someAgg.max.script]: definition for this key is missing"` - ); - }); + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.max.script]: definition for this key is missing"` + ); + }); - it('throws an error when using a script attribute in a nested aggregation', () => { - const aggregations: AggsMap = { - someAgg: { - min: { - field: 'foo.attributes.bytes', - }, - aggs: { - nested: { - max: { - field: 'foo.attributes.bytes', - script: 'This is a bad script', - }, + it('throws an error when using a script attribute in a nested aggregation', () => { + const aggregations: AggsMap = { + someAgg: { + min: { + field: 'foo.attributes.bytes', + }, + aggs: { + nested: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', }, }, }, - }; + }, + }; - expect(() => { - validateAndConvertAggregations(['foo'], aggregations, mockMappings); - }).toThrowErrorMatchingInlineSnapshot( - `"[someAgg.aggs.nested.max.script]: definition for this key is missing"` - ); - }); + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.aggs.nested.max.script]: definition for this key is missing"` + ); }); }); From eda6b743c1ac20a487c466f6542d4c6ac36322c0 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 7 Apr 2021 21:26:38 +0200 Subject: [PATCH 45/52] add some unit tests --- .../lib/aggregations/validation.test.ts | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts index 6a1d2cef532ff5..8a7c1c3719eb0f 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -70,7 +70,7 @@ const mockMappings = { }; describe('validateAndConvertAggregations', () => { - it('Validates a simple aggregations', () => { + it('validates a simple aggregations', () => { expect( validateAndConvertAggregations( ['foo'], @@ -182,7 +182,7 @@ describe('validateAndConvertAggregations', () => { }); }); - it('rewrites the `field` name when valid', () => { + it('rewrites type attributes when valid', () => { const aggregations: AggsMap = { average: { avg: { @@ -201,6 +201,25 @@ describe('validateAndConvertAggregations', () => { }); }); + it('rewrites root attributes when valid', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.updated_at', + missing: 10, + }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + avg: { + field: 'updated_at', + missing: 10, + }, + }, + }); + }); + it('throws an error when the `field` name is not using attributes path', () => { const aggregations: AggsMap = { average: { @@ -233,6 +252,22 @@ describe('validateAndConvertAggregations', () => { ); }); + it('throws an error when the attribute path is referencing an invalid root field', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.bad_root', + missing: 10, + }, + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.bad_root"` + ); + }); + it('rewrites the `field` name even when nested', () => { const aggregations: AggsMap = { average: { From 5d0e3522d8b704a67595d1d8d19544a1d4f131eb Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 09:05:35 +0200 Subject: [PATCH 46/52] adapt FTR assertions --- test/api_integration/apis/saved_objects/find.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index d9ffb63662abdf..5a8e706d3e4980 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -293,7 +293,7 @@ export default function ({ getService }: FtrProviderContext) { })); }); - describe('with a aggs', () => { + describe('using aggregations', () => { it('should return 200 with a valid response', async () => await supertest .get( @@ -331,7 +331,8 @@ export default function ({ getService }: FtrProviderContext) { .then((resp) => { expect(resp.body).to.eql({ error: 'Bad Request', - message: 'Invalid aggregation: This type dashboard is not allowed: Bad Request', + message: + 'Invalid aggregation: [type_count.max.field] Invalid attribute path: dashboard.attributes.version: Bad Request', statusCode: 400, }); })); From 8d9e41fc86dc42552d746cbacddc655fa6e49fbb Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 09:59:01 +0200 Subject: [PATCH 47/52] update doc --- docs/api/saved-objects/find.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/saved-objects/find.asciidoc b/docs/api/saved-objects/find.asciidoc index 556048a19e72a2..f04aeb84206207 100644 --- a/docs/api/saved-objects/find.asciidoc +++ b/docs/api/saved-objects/find.asciidoc @@ -60,7 +60,7 @@ experimental[] Retrieve a paginated set of {kib} saved objects by various condit `aggs`:: (Optional, string) **experimental** An aggregation structure, serialized as a string. The field format is similar to `filter`, meaning that to use a saved object type attribute in the aggregation, the `savedObjectType.attributes.title`: "myTitle"` format - must be used. + must be used. For root fields, the syntax is `savedObjectType.rootField` NOTE: As objects change in {kib}, the results on each page of the response also change. Use the find API for traditional paginated results, but avoid using it to export large amounts of data. From 5c6d3aca2fb06e90717286be32679e743b5abebd Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 10:42:02 +0200 Subject: [PATCH 48/52] fix doc --- src/core/server/saved_objects/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 9ae8793b9ca0a5..74acd9141c5be8 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -117,7 +117,7 @@ export interface SavedObjectsFindOptions { defaultSearchOperator?: 'AND' | 'OR'; filter?: string | KueryNode; /** - * Specify an Elasticsearch aggregation to perform. + * A record of aggregations to perform. * The API currently only supports a limited set of metrics and bucket aggregation types. * Additional aggregation types can be contributed to Core. * From c167987a3857ad4b71794bf8ac5e204e9658dc1e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 10:45:00 +0200 Subject: [PATCH 49/52] doc again --- src/core/server/saved_objects/types.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 74acd9141c5be8..d3bfdcc6923dcf 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -122,11 +122,19 @@ export interface SavedObjectsFindOptions { * Additional aggregation types can be contributed to Core. * * @example + * Aggregating on SO attribute field * ```ts * const aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; * return client.find({ type: 'dashboard', aggs }) * ``` * + * @example + * Aggregating on SO root field + * ```ts + * const aggs = { latest_update: { max: { field: 'dashboard.updated_at' } } }; + * return client.find({ type: 'dashboard', aggs }) + * ``` + * * @alpha */ aggs?: Record; From 5239a44d41169fa8f6d1e33bc72878ea62aa7c32 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 10:50:13 +0200 Subject: [PATCH 50/52] cleanup test names --- test/api_integration/apis/saved_objects/find.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index 5a8e706d3e4980..a01562861e606c 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -294,7 +294,7 @@ export default function ({ getService }: FtrProviderContext) { }); describe('using aggregations', () => { - it('should return 200 with a valid response', async () => + it('should return 200 with valid response for a valid aggregation', async () => await supertest .get( `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( @@ -318,7 +318,7 @@ export default function ({ getService }: FtrProviderContext) { }); })); - it('wrong type should return 400 with Bad Request', async () => + it('should return a 400 when referencing an invalid SO attribute', async () => await supertest .get( `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( @@ -337,7 +337,7 @@ export default function ({ getService }: FtrProviderContext) { }); })); - it('adding a wrong attributes should return 400 with Bad Request', async () => + it('should return a 400 when using a forbidden aggregation option', async () => await supertest .get( `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( From cb0327c923eeaca1fba04b488d778fb6bdc10b93 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 8 Apr 2021 11:08:49 +0200 Subject: [PATCH 51/52] improve tsdoc on validation functions --- .../service/lib/aggregations/validation.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index 899061c7797a17..6dad428efaf79f 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -28,10 +28,12 @@ interface ValidationContext { } /** - * Validates an aggregation structure against the declared mapping and - * aggregation schema, and rewrite the attribute fields from the - * `{type}.attributes.{attribute}` (KQL-like) syntax to the underlying - * `{type}.{attribute} format.` + * Validate an aggregation structure against the declared mappings and + * aggregation schemas, and rewrite the attribute fields using the KQL-like syntax + * - `{type}.attributes.{attribute}` to `{type}.{attribute}` + * - `{type}.{rootField}` to `{rootField}` + * + * throws on the first validation error if any is encountered. */ export const validateAndConvertAggregations = ( allowedTypes: string[], @@ -46,12 +48,9 @@ export const validateAndConvertAggregations = ( }; /** - * Validates a record of aggregation containers, + * Validate a record of aggregation containers, * Which can either be the root level aggregations (`SearchRequest.body.aggs`) * Or a nested record of aggregation (`SearchRequest.body.aggs.myAggregation.aggs`) - * - * @param aggregations - * @param context */ const validateAggregations = ( aggregations: Record, @@ -66,11 +65,8 @@ const validateAggregations = ( }; /** - * Validates an aggregation container, e.g an entry of `SearchRequest.body.aggs`, or - * from a nested aggregation record. - * - * @param aggregation - * @param context + * Validate an aggregation container, e.g an entry of `SearchRequest.body.aggs`, or + * from a nested aggregation record, including its potential nested aggregations. */ const validateAggregation = ( aggregation: estypes.AggregationContainer, @@ -91,6 +87,10 @@ const validateAggregation = ( return container; }; +/** + * Validates root-level aggregation of given aggregation container + * (ignoring its nested aggregations) + */ const validateAggregationContainer = ( container: estypes.AggregationContainer, context: ValidationContext From fba668ee2af49cc5c6033f7d9be17d223d1b5787 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 12 Apr 2021 10:01:15 +0200 Subject: [PATCH 52/52] perf nit --- .../saved_objects/service/lib/aggregations/validation.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts index 6dad428efaf79f..a2fd392183132f 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/validation.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -57,10 +57,8 @@ const validateAggregations = ( context: ValidationContext ) => { return Object.entries(aggregations).reduce((memo, [aggrName, aggrContainer]) => { - return { - ...memo, - [aggrName]: validateAggregation(aggrContainer, childContext(context, aggrName)), - }; + memo[aggrName] = validateAggregation(aggrContainer, childContext(context, aggrName)); + return memo; }, {} as Record); };