[ML] List data recognizer modules (elastic#30935) (elastic#34363)

* [ML] [WIP] List data recognizer modules

* making module id optional

* sort modules

* removing list_module endpoint

* removing list_modules call

* listed modules include saveObjects

* adding tests

x-pack/plugins/ml/public/services/ml_api_service/index.js

@@ -247,6 +247,13 @@ export const ml = {
  });
  },
 
+ listDataRecognizerModules() {
+ return http({
+ url: `${basePath}/modules/get_module`,
+ method: 'GET'
+ });
+ },
+
  getDataRecognizerModule(obj) {
  return http({
  url: `${basePath}/modules/get_module/${obj.moduleId}`,

x-pack/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+
+import expect from '@kbn/expect';
+import { DataRecognizer } from '../data_recognizer';
+
+describe('ML - data recognizer', () => {
+ const dr = new DataRecognizer({});
+
+ const moduleIds = [
+ 'apache_ecs',
+ 'apm_transaction',
+ 'auditbeat_process_docker_ecs',
+ 'auditbeat_process_hosts_ecs',
+ 'nginx_ecs',
+ ];
+
+ // check all module IDs are the same as the list above
+ it('listModules - check all module IDs', async (done) => {
+ const modules = await dr.listModules();
+ const ids = modules.map(m => m.id);
+ expect(ids.join()).to.equal(moduleIds.join());
+ done();
+ });
+
+
+ it('getModule - load a single module', async (done) => {
+ const module = await dr.getModule(moduleIds[0]);
+ expect(module.id).to.equal(moduleIds[0]);
+ done();
+ });
+
+});

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js

@@ -150,6 +150,20 @@ export class DataRecognizer {
  return (resp.hits.total !== 0);
  }
 
+ async listModules() {
+ const manifestFiles = await this.loadManifestFiles();
+ const ids = manifestFiles
+ .map(({ json }) => json.id)
+ .sort((a, b) => a.localeCompare(b)); // sort as json files are read from disk and could be in any order.
+
+ const modules = [];
+ for (let i = 0; i < ids.length; i++) {
+ const module = await this.getModule(ids[i]);
+ modules.push(module);
+ }
+ return modules;
+ }
+
  // called externally by an endpoint
  // supplying an optional prefix will add the prefix
  // to the job and datafeed configs
@@ -224,6 +238,7 @@ export class DataRecognizer {
  }
 
  return {
+ ...manifestJSON,
  jobs,
  datafeeds,
  kibana

x-pack/plugins/ml/server/routes/modules.js

@@ -18,7 +18,11 @@ function recognize(callWithRequest, indexPatternTitle) {
 
 function getModule(callWithRequest, moduleId) {
  const dr = new DataRecognizer(callWithRequest);
- return dr.getModule(moduleId);
+ if (moduleId === undefined) {
+ return dr.listModules();
+ } else {
+ return dr.getModule(moduleId);
+ }
 }
 
 function saveModuleItems(
@@ -66,7 +70,7 @@ export function dataRecognizer(server, commonRouteConfig) {
 
  server.route({
  method: 'GET',
- path: '/api/ml/modules/get_module/{moduleId}',
+ path: '/api/ml/modules/get_module/{moduleId?}',
  handler(request) {
  const callWithRequest = callWithRequestFactory(server, request);
  const moduleId = request.params.moduleId;