Timelion query language support for scripted fields (#14700)

* update run REST API to fetch index pattern and use script for scripted fields

* allow scripted fields for elasticsearch split argument

* add test cases for scripted fields to elasticsearch test

* update help text for .es() index argument

* move scripted field check into common file, remove stop characters from index pattern title

* use space instead of nothing for replacing stop chars

* wrap index name in quotes instead of removing dash character

src/core_plugins/timelion/server/series_functions/__tests__/es.js

@@ -13,7 +13,7 @@ import { expect } from 'chai';
 import sinon from 'sinon';
 import invoke from './helpers/invoke_series_fn.js';
 
-function stubResponse(response) {
+function stubRequestAndServer(response, indexPatternSavedObjects = []) {
  return {
  server: {
  plugins: {
@@ -25,6 +25,17 @@ function stubResponse(response) {
  })
  }
  }
+ },
+ request: {
+ getSavedObjectsClient: function () {
+ return {
+ find: function () {
+ return Promise.resolve({
+ saved_objects: indexPatternSavedObjects
+ });
+ }
+ };
+ }
  }
  };
 }
@@ -34,7 +45,7 @@ describe(filename, () => {
 
  describe('seriesList processor', () => {
  it('throws an error then the index is missing', () => {
- tlConfig = stubResponse({
+ tlConfig = stubRequestAndServer({
  _shards: { total: 0 }
  });
  return invoke(es, [5], tlConfig)
@@ -45,7 +56,7 @@ describe(filename, () => {
  });
 
  it('returns a seriesList', () => {
- tlConfig = stubResponse(esResponse);
+ tlConfig = stubRequestAndServer(esResponse);
  return invoke(es, [5], tlConfig)
  .then((r) => {
  expect(r.output.type).to.eql('seriesList');
@@ -92,16 +103,36 @@ describe(filename, () => {
  });
 
  describe('metric aggs', () => {
+ const emptyScriptedFields = [];
+
  it('adds a metric agg for each metric', () => {
  config.metric = ['sum:beer', 'avg:bytes'];
- agg = createDateAgg(config, tlConfig);
+ agg = createDateAgg(config, tlConfig, emptyScriptedFields);
  expect(agg.time_buckets.aggs['sum(beer)']).to.eql({ sum: { field: 'beer' } });
  expect(agg.time_buckets.aggs['avg(bytes)']).to.eql({ avg: { field: 'bytes' } });
  });
 
+ it('adds a scripted metric agg for each scripted metric', () => {
+ config.metric = ['avg:scriptedBytes'];
+ const scriptedFields = [{
+ name: 'scriptedBytes',
+ script: 'doc["bytes"].value',
+ lang: 'painless'
+ }];
+ agg = createDateAgg(config, tlConfig, scriptedFields);
+ expect(agg.time_buckets.aggs['avg(scriptedBytes)']).to.eql({
+ avg: {
+ script: {
+ inline: 'doc["bytes"].value',
+ lang: 'painless'
+ }
+ }
+ });
+ });
+
  it('has a special `count` metric that uses a script', () => {
  config.metric = ['count'];
- agg = createDateAgg(config, tlConfig);
+ agg = createDateAgg(config, tlConfig, emptyScriptedFields);
  expect(agg.time_buckets.aggs.count.bucket_script).to.be.an('object');
  expect(agg.time_buckets.aggs.count.bucket_script.buckets_path).to.eql('_count');
  });
@@ -110,6 +141,7 @@ describe(filename, () => {
 
  describe('buildRequest', () => {
  const fn = buildRequest;
+ const emptyScriptedFields = [];
  let tlConfig;
  let config;
  beforeEach(() => {
@@ -124,20 +156,20 @@ describe(filename, () => {
 
  it('sets the index on the request', () => {
  config.index = 'beer';
- const request = fn(config, tlConfig);
+ const request = fn(config, tlConfig, emptyScriptedFields);
 
  expect(request.index).to.equal('beer');
  });
 
  it('always sets body.size to 0', () => {
- const request = fn(config, tlConfig);
+ const request = fn(config, tlConfig, emptyScriptedFields);
 
  expect(request.body.size).to.equal(0);
  });
 
  it('creates a filters agg that contains each of the queries passed', () => {
  config.q = ['foo', 'bar'];
- const request = fn(config, tlConfig);
+ const request = fn(config, tlConfig, emptyScriptedFields);
 
  expect(request.body.aggs.q.meta.type).to.equal('split');
 
@@ -169,20 +201,20 @@ describe(filename, () => {
 
  it('adds the contents of payload.extended.es.filter to a filter clause of the bool', () => {
  config.kibana = true;
- const request = fn(config, tlConfig);
+ const request = fn(config, tlConfig, emptyScriptedFields);
  const filter = request.body.query.bool.filter.bool;
  expect(filter.must.length).to.eql(1);
  expect(filter.must_not.length).to.eql(2);
  });
 
  it('does not include filters if config.kibana = false', () => {
  config.kibana = false;
- const request = fn(config, tlConfig);
+ const request = fn(config, tlConfig, emptyScriptedFields);
  expect(request.body.query.bool.filter).to.eql(undefined);
  });
 
  it('adds a time filter to the bool querys must clause', () => {
- let request = fn(config, tlConfig);
+ let request = fn(config, tlConfig, emptyScriptedFields);
  expect(request.body.query.bool.must.length).to.eql(1);
  expect(request.body.query.bool.must[0]).to.eql({ range: { '@timestamp': {
  lte: 5,
@@ -191,24 +223,59 @@ describe(filename, () => {
  } } });
 
  config.kibana = true;
- request = fn(config, tlConfig);
+ request = fn(config, tlConfig, emptyScriptedFields);
  expect(request.body.query.bool.must.length).to.eql(1);
  });
  });
 
- it('config.split adds terms aggs, in order, under the filters agg', () => {
- config.split = ['beer:5', 'wine:10'];
- const request = fn(config, tlConfig);
+ describe('config.split', () => {
+ it('adds terms aggs, in order, under the filters agg', () => {
+ config.split = ['beer:5', 'wine:10'];
+ const request = fn(config, tlConfig, emptyScriptedFields);
+
+ const aggs = request.body.aggs.q.aggs;
+
+ expect(aggs.beer.meta.type).to.eql('split');
+ expect(aggs.beer.terms.field).to.eql('beer');
+ expect(aggs.beer.terms.size).to.eql(5);
 
- const aggs = request.body.aggs.q.aggs;
+ expect(aggs.beer.aggs.wine.meta.type).to.eql('split');
+ expect(aggs.beer.aggs.wine.terms.field).to.eql('wine');
+ expect(aggs.beer.aggs.wine.terms.size).to.eql(10);
+ });
+
+ it('adds scripted terms aggs, in order, under the filters agg', () => {
+ config.split = ['scriptedBeer:5', 'scriptedWine:10'];
+ const scriptedFields = [
+ {
+ name: 'scriptedBeer',
+ script: 'doc["beer"].value',
+ lang: 'painless'
+ },
+ {
+ name: 'scriptedWine',
+ script: 'doc["wine"].value',
+ lang: 'painless'
+ }
+ ];
+ const request = fn(config, tlConfig, scriptedFields);
 
- expect(aggs.beer.meta.type).to.eql('split');
- expect(aggs.beer.terms.field).to.eql('beer');
- expect(aggs.beer.terms.size).to.eql(5);
+ const aggs = request.body.aggs.q.aggs;
 
- expect(aggs.beer.aggs.wine.meta.type).to.eql('split');
- expect(aggs.beer.aggs.wine.terms.field).to.eql('wine');
- expect(aggs.beer.aggs.wine.terms.size).to.eql(10);
+ expect(aggs.scriptedBeer.meta.type).to.eql('split');
+ expect(aggs.scriptedBeer.terms.script).to.eql({
+ inline: 'doc["beer"].value',
+ lang: 'painless'
+ });
+ expect(aggs.scriptedBeer.terms.size).to.eql(5);
+
+ expect(aggs.scriptedBeer.aggs.scriptedWine.meta.type).to.eql('split');
+ expect(aggs.scriptedBeer.aggs.scriptedWine.terms.script).to.eql({
+ inline: 'doc["wine"].value',
+ lang: 'painless'
+ });
+ expect(aggs.scriptedBeer.aggs.scriptedWine.terms.size).to.eql(10);
+ });
  });
  });
 

src/core_plugins/timelion/server/series_functions/es/index.js

@@ -27,7 +27,7 @@ export default new Datasource('es', {
  {
  name: 'index',
  types: ['string', 'null'],
- help: 'Index to query, wildcards accepted'
+ help: 'Index to query, wildcards accepted. Provide Index Pattern name for scripted field support.'
  },
  {
  name: 'timefield',
@@ -47,7 +47,7 @@ export default new Datasource('es', {
  ],
  help: 'Pull data from an elasticsearch instance',
  aliases: ['elasticsearch'],
- fn: function esFn(args, tlConfig) {
+ fn: async function esFn(args, tlConfig) {
 
  const config = _.defaults(_.clone(args.byName), {
  q: '*',
@@ -59,16 +59,31 @@ export default new Datasource('es', {
  fit: 'nearest'
  });
 
- const { callWithRequest } = tlConfig.server.plugins.elasticsearch.getCluster('data');
+ const findResp = await tlConfig.request.getSavedObjectsClient().find({
+ type: 'index-pattern',
+ fields: ['title', 'fields'],
+ search: `"${config.index}"`,
+ search_fields: ['title']
+ });
+ const indexPatternSavedObject = findResp.saved_objects.find(savedObject => {
+ return savedObject.attributes.title === config.index;
+ });
+ let scriptedFields = [];
+ if (indexPatternSavedObject) {
+ const fields = JSON.parse(indexPatternSavedObject.attributes.fields);
+ scriptedFields = fields.filter(field => {
+ return field.scripted;
+ });
+ }
 
- const body = buildRequest(config, tlConfig);
+ const body = buildRequest(config, tlConfig, scriptedFields);
 
- return callWithRequest(tlConfig.request, 'search', body).then(function (resp) {
-  if (!resp._shards.total) throw new Error('Elasticsearch index not found: ' + config.index);
-  return {
-  type: 'seriesList',
-  list: toSeriesList(resp.aggregations, config)
- };
- });
+ const { callWithRequest } = tlConfig.server.plugins.elasticsearch.getCluster('data');
+ const resp = await callWithRequest(tlConfig.request, 'search', body);
+ if (!resp._shards.total) throw new Error('Elasticsearch index not found: ' + config.index);
+ return {
+ type: 'seriesList',
+ list: toSeriesList(resp.aggregations, config)
+ };
  }
 });

src/core_plugins/timelion/server/series_functions/es/lib/agg_body.js

@@ -0,0 +1,20 @@
+
+export function buildAggBody(fieldName, scriptedFields) {
+
+ const scriptedField = scriptedFields.find(field => {
+ return field.name === fieldName;
+ });
+
+ if (scriptedField) {
+ return {
+ script: {
+ inline: scriptedField.script,
+ lang: scriptedField.lang
+ }
+ };
+ }
+
+ return {
+ field: fieldName
+ };
+}

src/core_plugins/timelion/server/series_functions/es/lib/build_request.js

@@ -1,7 +1,8 @@
 import _ from 'lodash';
+import { buildAggBody } from './agg_body';
 import createDateAgg from './create_date_agg';
 
-export default function buildRequest(config, tlConfig) {
+export default function buildRequest(config, tlConfig, scriptedFields) {
 
  const bool = { must: [] };
 
@@ -31,12 +32,11 @@ export default function buildRequest(config, tlConfig) {
  _.each(config.split, function (clause) {
  clause = clause.split(':');
  if (clause[0] && clause[1]) {
+ const termsAgg = buildAggBody(clause[0], scriptedFields);
+ termsAgg.size = parseInt(clause[1], 10);
  aggCursor[clause[0]] = {
  meta: { type: 'split' },
- terms: {
- field: clause[0],
- size: parseInt(clause[1], 10)
- },
+ terms: termsAgg,
  aggs: {}
  };
  aggCursor = aggCursor[clause[0]].aggs;
@@ -45,7 +45,7 @@ export default function buildRequest(config, tlConfig) {
  }
  });
 
- _.assign(aggCursor, createDateAgg(config, tlConfig));
+ _.assign(aggCursor, createDateAgg(config, tlConfig, scriptedFields));
 
 
  return {

src/core_plugins/timelion/server/series_functions/es/lib/create_date_agg.js

@@ -1,6 +1,7 @@
 import _ from 'lodash';
+import { buildAggBody } from './agg_body';
 
-export default function createDateAgg(config, tlConfig) {
+export default function createDateAgg(config, tlConfig, scriptedFields) {
  const dateAgg = {
  time_buckets: {
  meta: { type: 'time_buckets' },
@@ -32,7 +33,7 @@ export default function createDateAgg(config, tlConfig) {
  } else if (metric[0] && metric[1]) {
  const metricName = metric[0] + '(' + metric[1] + ')';
  dateAgg.time_buckets.aggs[metricName] = {};
- dateAgg.time_buckets.aggs[metricName][metric[0]] = { field: metric[1] };
+ dateAgg.time_buckets.aggs[metricName][metric[0]] = buildAggBody(metric[1], scriptedFields);
  } else {
  throw new Error ('`metric` requires metric:field or simply count');
  }