[Security Solution] [Sourcerer] [Feature Branch] Update to use Kibana Data Views

src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -5,7 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
 import { IndexPatternsFetcher } from '.';
 import { ElasticsearchClient } from 'kibana/server';
 import * as indexNotFoundException from './index_not_found_exception.json';
@@ -15,36 +14,36 @@ describe('Index Pattern Fetcher - server', () => {
  let esClient: ElasticsearchClient;
  const emptyResponse = {
  body: {
- count: 0,
+ indices: [],
  },
  };
  const response = {
  body: {
- count: 1115,
+ indices: ['b'],
+ fields: [{ name: 'foo' }, { name: 'bar' }, { name: 'baz' }],
  },
  };
  const patternList = ['a', 'b', 'c'];
  beforeEach(() => {
+ jest.clearAllMocks();
  esClient = {
- count: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
+ fieldCaps: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
  } as unknown as ElasticsearchClient;
  indexPatterns = new IndexPatternsFetcher(esClient);
  });
-
  it('Removes pattern without matching indices', async () => {
  const result = await indexPatterns.validatePatternListActive(patternList);
  expect(result).toEqual(['b', 'c']);
  });
-
  it('Returns all patterns when all match indices', async () => {
  esClient = {
- count: jest.fn().mockResolvedValue(response),
+ fieldCaps: jest.fn().mockResolvedValue(response),
  } as unknown as ElasticsearchClient;
  indexPatterns = new IndexPatternsFetcher(esClient);
  const result = await indexPatterns.validatePatternListActive(patternList);
  expect(result).toEqual(patternList);
  });
- it('Removes pattern when "index_not_found_exception" error is thrown', async () => {
+ it('Removes pattern when error is thrown', async () => {
  class ServerError extends Error {
  public body?: Record<string, any>;
  constructor(
@@ -56,9 +55,8 @@ describe('Index Pattern Fetcher - server', () => {
  this.body = errBody;
  }
  }
-
  esClient = {
- count: jest
+ fieldCaps: jest
  .fn()
  .mockResolvedValueOnce(response)
  .mockRejectedValue(
@@ -69,4 +67,22 @@ describe('Index Pattern Fetcher - server', () => {
  const result = await indexPatterns.validatePatternListActive(patternList);
  expect(result).toEqual([patternList[0]]);
  });
+ it('When allowNoIndices is false, run validatePatternListActive', async () => {
+ const fieldCapsMock = jest.fn();
+ esClient = {
+ fieldCaps: fieldCapsMock.mockResolvedValue(response),
+ } as unknown as ElasticsearchClient;
+ indexPatterns = new IndexPatternsFetcher(esClient);
+ await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+ expect(fieldCapsMock.mock.calls).toHaveLength(4);
+ });
+ it('When allowNoIndices is true, do not run validatePatternListActive', async () => {
+ const fieldCapsMock = jest.fn();
+ esClient = {
+ fieldCaps: fieldCapsMock.mockResolvedValue(response),
+ } as unknown as ElasticsearchClient;
+ indexPatterns = new IndexPatternsFetcher(esClient, true);
+ await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+ expect(fieldCapsMock.mock.calls).toHaveLength(1);
+ });
 });

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -36,12 +36,10 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
  private elasticsearchClient: ElasticsearchClient;
  private allowNoIndices: boolean;
-
  constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
  this.elasticsearchClient = elasticsearchClient;
  this.allowNoIndices = allowNoIndices;
  }
-
  /**
  * Get a list of field objects for an index pattern that may contain wildcards
  *
@@ -60,23 +58,22 @@ export class IndexPatternsFetcher {
  }): Promise<FieldDescriptor[]> {
  const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
  const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
+ const allowNoIndices = fieldCapsOptions
+ ? fieldCapsOptions.allow_no_indices
+ : this.allowNoIndices;
  let patternListActive: string[] = patternList;
  // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
- if (patternList.length > 1) {
+ if (patternList.length > 1 && !allowNoIndices) {
  patternListActive = await this.validatePatternListActive(patternList);
  }
  const fieldCapsResponse = await getFieldCapabilities(
  this.elasticsearchClient,
- // if none of the patterns are active, pass the original list to get an error
- patternListActive.length > 0 ? patternListActive : patternList,
+ patternListActive,
  metaFields,
  {
- allow_no_indices: fieldCapsOptions
- ? fieldCapsOptions.allow_no_indices
- : this.allowNoIndices,
+ allow_no_indices: allowNoIndices,
  }
  );
-
  if (type === 'rollup' && rollupIndex) {
  const rollupFields: FieldDescriptor[] = [];
  const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -87,13 +84,11 @@ export class IndexPatternsFetcher {
  ).body
  )[rollupIndex].aggs;
  const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
-
  // Keep meta fields
  metaFields!.forEach(
  (field: string) =>
  fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
  );
-
  return mergeCapabilitiesWithFields(
  rollupIndexCapabilities,
  fieldCapsResponseObj,
@@ -137,23 +132,20 @@ export class IndexPatternsFetcher {
  async validatePatternListActive(patternList: string[]) {
  const result = await Promise.all(
  patternList
- .map((pattern) =>
- this.elasticsearchClient.count({
- index: pattern,
- })
- )
- .map((p) =>
- p.catch((e) => {
- if (e.body.error.type === 'index_not_found_exception') {
- return { body: { count: 0 } };
- }
- throw e;
- })
- )
+ .map(async (index) => {
+ const searchResponse = await this.elasticsearchClient.fieldCaps({
+ index,
+ fields: '_id',
+ ignore_unavailable: true,
+ allow_no_indices: false,
+ });
+ return searchResponse.body.indices.length > 0;
+ })
+ .map((p) => p.catch(() => false))
  );
  return result.reduce(
- (acc: string[], { body: { count } }, patternListIndex) =>
- count > 0 ? [...acc, patternList[patternListIndex]] : acc,
+ (acc: string[], isValid, patternListIndex) =>
+ isValid ? [...acc, patternList[patternListIndex]] : acc,
  []
  );
  }

x-pack/plugins/observability/public/pages/alerts/alerts_table_t_grid.tsx

@@ -412,6 +412,8 @@ export function AlertsTableTGrid(props: AlertsTableTGridProps) {
  },
  renderCellValue: getRenderCellValue({ setFlyoutAlert }),
  rowRenderers: NO_ROW_RENDER,
+ // TODO: implement Kibana data view runtime fields in observability
+ runtimeMappings: {},
  start: rangeFrom,
  setRefetch,
  sort: [

x-pack/plugins/security_solution/common/constants.ts

@@ -10,9 +10,19 @@ import { ENABLE_ITOM } from '../../actions/server/constants/connectors';
 import type { TransformConfigSchema } from './transforms/types';
 import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
 import { METADATA_TRANSFORMS_PATTERN } from './endpoint/constants';
+import { SavedObjectReference } from '../../../../src/core/types';
+import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../../../src/plugins/data/common';
+
+/**
+ * as const
+ *
+ * The const assertion ensures that type widening does not occur
+ * https://mariusschulz.com/blog/literal-type-widening-in-typescript
+ * Please follow this convention when adding to this file
+ */
 
 export const APP_ID = 'securitySolution' as const;
-export const APP_UI_ID = 'securitySolutionUI';
+export const APP_UI_ID = 'securitySolutionUI' as const;
 export const CASES_FEATURE_ID = 'securitySolutionCases' as const;
 export const SERVER_APP_ID = 'siem' as const;
 export const APP_NAME = 'Security' as const;
@@ -26,6 +36,8 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const;
 export const DEFAULT_DARK_MODE = 'theme:darkMode' as const;
 export const DEFAULT_INDEX_KEY = 'securitySolution:defaultIndex' as const;
 export const DEFAULT_NUMBER_FORMAT = 'format:number:defaultPattern' as const;
+export const DEFAULT_DATA_VIEW_ID = 'security-solution' as const;
+export const DEFAULT_TIME_FIELD = '@timestamp' as const;
 export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults' as const;
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults' as const;
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const;
@@ -51,7 +63,6 @@ export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges' as const
 export const DEFAULT_TRANSFORMS = 'securitySolution:transforms' as const;
 export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled' as const;
 export const GLOBAL_HEADER_HEIGHT = 96 as const; // px
-export const GLOBAL_HEADER_HEIGHT_WITH_GLOBAL_BANNER = 128 as const; // px
 export const FILTERS_GLOBAL_HEIGHT = 109 as const; // px
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled' as const;
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51' as const;
@@ -152,6 +163,12 @@ export const DEFAULT_INDEX_PATTERN = [
  'winlogbeat-*',
 ];
 
+export const defaultDataViewRef: SavedObjectReference = {
+ id: DEFAULT_DATA_VIEW_ID,
+ name: 'dataViewId',
+ type: DATA_VIEW_SAVED_OBJECT_TYPE,
+};
+
 export const DEFAULT_INDEX_PATTERN_EXPERIMENTAL = [
  // TODO: Steph/ueba TEMP for testing UEBA data
  'ml_host_risk_score_*',
@@ -256,6 +273,7 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged` as const;
 
 export const NOTE_URL = '/api/note' as const;
 export const PINNED_EVENT_URL = '/api/pinned_event' as const;
+export const SOURCERER_API_URL = '/api/sourcerer' as const;
 
 /**
  * Default signals index key for kibana.dev.yml
@@ -343,7 +361,7 @@ export const ELASTIC_NAME = 'estc' as const;
 
 export const METADATA_TRANSFORM_STATS_URL = `/api/transform/transforms/${METADATA_TRANSFORMS_PATTERN}/_stats`;
 
-export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_';
+export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_' as const;
 
 export const TRANSFORM_STATES = {
  ABORTING: 'aborting',

x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts

@@ -5,79 +5,15 @@
  * 2.0.
  */
 
-import type { IFieldSubType } from '@kbn/es-query';
-
-import type {
- IEsSearchRequest,
- IEsSearchResponse,
- IIndexPattern,
-} from '../../../../../../src/plugins/data/common';
-import type { DocValueFields, Maybe } from '../common';
-
-interface FieldInfo {
- category: string;
- description?: string;
- example?: string | number;
- format?: string;
- name: string;
- type?: string;
-}
-
-export interface IndexField {
- /** Where the field belong */
- category: string;
- /** Example of field's value */
- example?: Maybe<string | number>;
- /** whether the field's belong to an alias index */
- indexes: Array<Maybe<string>>;
- /** The name of the field */
- name: string;
- /** The type of the field's values as recognized by Kibana */
- type: string;
- /** Whether the field's values can be efficiently searched for */
- searchable: boolean;
- /** Whether the field's values can be aggregated */
- aggregatable: boolean;
- /** Description of the field */
- description?: Maybe<string>;
- format?: Maybe<string>;
- /** the elastic type as mapped in the index */
- esTypes?: string[];
- subType?: IFieldSubType;
- readFromDocValues: boolean;
-}
-
-export type BeatFields = Record<string, FieldInfo>;
-
-export interface IndexFieldsStrategyRequest extends IEsSearchRequest {
- indices: string[];
- onlyCheckIfIndicesExist: boolean;
-}
-
-export interface IndexFieldsStrategyResponse extends IEsSearchResponse {
- indexFields: IndexField[];
- indicesExist: string[];
-}
-
-export interface BrowserField {
- aggregatable: boolean;
- category: string;
- description: string | null;
- example: string | number | null;
- fields: Readonly<Record<string, Partial<BrowserField>>>;
- format: string;
- indexes: string[];
- name: string;
- searchable: boolean;
- type: string;
- subType?: IFieldSubType;
-}
-
-export type BrowserFields = Readonly<Record<string, Partial<BrowserField>>>;
-
-export const EMPTY_BROWSER_FIELDS = {};
-export const EMPTY_DOCVALUE_FIELD: DocValueFields[] = [];
-export const EMPTY_INDEX_PATTERN: IIndexPattern = {
- fields: [],
- title: '',
-};
+export {
+ FieldInfo,
+ IndexField,
+ BeatFields,
+ IndexFieldsStrategyRequest,
+ IndexFieldsStrategyResponse,
+ BrowserField,
+ BrowserFields,
+ EMPTY_BROWSER_FIELDS,
+ EMPTY_DOCVALUE_FIELD,
+ EMPTY_INDEX_PATTERN,
+} from '../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/events/last_event_time/index.ts

@@ -10,6 +10,7 @@ export { LastEventIndexKey } from '../../../../../../timelines/common';
 export type {
  LastTimeDetails,
  TimelineEventsLastEventTimeStrategyResponse,
+ TimelineKpiStrategyRequest,
  TimelineKpiStrategyResponse,
  TimelineEventsLastEventTimeRequestOptions,
 } from '../../../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { IEsSearchRequest } from '../../../../../../src/plugins/data/common';
 import { ESQuery } from '../../typed_json';
 import {
@@ -41,6 +42,7 @@ export interface TimelineRequestBasicOptions extends IEsSearchRequest {
  defaultIndex: string[];
  docValueFields?: DocValueFields[];
  factoryQueryType?: TimelineFactoryQueryTypes;
+ runtimeMappings: MappingRuntimeFields;
 }
 
 export interface TimelineRequestSortField<Field = string> extends SortField<Field> {
@@ -171,6 +173,7 @@ export interface SortTimelineInput {
 export interface TimelineInput {
  columns?: Maybe<ColumnHeaderInput[]>;
  dataProviders?: Maybe<DataProviderInput[]>;
+ dataViewId?: Maybe<string>;
  description?: Maybe<string>;
  eqlOptions?: Maybe<EqlOptionsInput>;
  eventType?: Maybe<string>;

x-pack/plugins/security_solution/common/test/index.ts

@@ -7,12 +7,12 @@
 
 // For the source of these roles please consult the PR these were introduced https:/elastic/kibana/pull/81866#issue-511165754
 export enum ROLES {
+ soc_manager = 'soc_manager',
  reader = 'reader',
  t1_analyst = 't1_analyst',
  t2_analyst = 't2_analyst',
  hunter = 'hunter',
  rule_author = 'rule_author',
- soc_manager = 'soc_manager',
  platform_engineer = 'platform_engineer',
  detections_admin = 'detections_admin',
 }