Enhance the ingest node simulate verbose output (elastic#60433)

This commit enhances the verbose output for the `_ingest/pipeline/_simulate?verbose` api. Specifically this adds the following: * the pipeline processor is now included in the output * the conditional (if) and result is now included in the output iff it was defined * a status field is always displayed. the possible values of status are * `success` - if the processor ran with out errors * `error` - if the processor ran but threw an error that was not ingored * `error_ignored` - if the processor ran but threw an error that was ingored * `skipped` - if the process did not run (currently only possible if the if condition evaluates to false) * `dropped` - if the the `drop` processor ran and dropped the document * a `processor_type` field for the type of processor (e.g. set, rename, etc.) * throw a better error if trying to simulate with a pipeline that does not exist closes elastic#56004 # Conflicts: # docs/reference/ingest/apis/simulate-pipeline.asciidoc
jakelandis · Aug 4, 2020 · 0edc03b · 0edc03b
1 parent 0587199
commit 0edc03b
Show file tree

Hide file tree

Showing 10 changed files with 507 additions and 217 deletions.
diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc
@@ -342,84 +342,84 @@ The API returns the following response:
 [source,console-result]
 ----
 {
- "docs": [
- {
- "processor_results": [
- {
- "doc": {
- "_id": "id",
- "_index": "index",
- "_type": "_doc",
- "_source": {
- "field2": "_value2",
- "foo": "bar"
- },
- "_ingest": {
- "timestamp": "2017-05-04T22:46:09.674Z",
- "pipeline": "_simulate_pipeline"
- }
- }
+ "docs" : [
+ {
+ "processor_results" : [
+ {
+ "processor_type" : "set",
+ "status" : "success",
+ "doc" : {
+ "_index" : "index",
+ "_source" : {
+ "field2" : "_value2",
+ "foo" : "bar"
  },
- {
- "doc": {
- "_id": "id",
- "_index": "index",
- "_type": "_doc",
- "_source": {
- "field3": "_value3",
- "field2": "_value2",
- "foo": "bar"
- },
- "_ingest": {
- "timestamp": "2017-05-04T22:46:09.675Z",
- "pipeline": "_simulate_pipeline"
- }
- }
+ "_ingest" : {
+ "pipeline" : "_simulate_pipeline",
+ "timestamp" : "2020-07-30T01:21:24.251836Z"
  }
- ]
- },
- {
- "processor_results": [
- {
- "doc": {
- "_id": "id",
- "_index": "index",
- "_type": "_doc",
- "_source": {
- "field2": "_value2",
- "foo": "rab"
- },
- "_ingest": {
- "timestamp": "2017-05-04T22:46:09.676Z",
- "pipeline": "_simulate_pipeline"
- }
- }
+ }
+ },
+ {
+ "processor_type" : "set",
+ "status" : "success",
+ "doc" : {
+ "_index" : "index",
+ "_source" : {
+ "field3" : "_value3",
+ "field2" : "_value2",
+ "foo" : "bar"
  },
- {
- "doc": {
- "_id": "id",
- "_index": "index",
- "_type": "_doc",
- "_source": {
- "field3": "_value3",
- "field2": "_value2",
- "foo": "rab"
- },
- "_ingest": {
- "timestamp": "2017-05-04T22:46:09.677Z",
- "pipeline": "_simulate_pipeline"
- }
- }
+ "_ingest" : {
+ "pipeline" : "_simulate_pipeline",
+ "timestamp" : "2020-07-30T01:21:24.251836Z"
  }
- ]
- }
- ]
+ }
+ }
+ ]
+ },
+ {
+ "processor_results" : [
+ {
+ "processor_type" : "set",
+ "status" : "success",
+ "doc" : {
+ "_index" : "index",
+ "_source" : {
+ "field2" : "_value2",
+ "foo" : "rab"
+ },
+ "_ingest" : {
+ "pipeline" : "_simulate_pipeline",
+ "timestamp" : "2020-07-30T01:21:24.251863Z"
+ }
+ }
+ },
+ {
+ "processor_type" : "set",
+ "status" : "success",
+ "doc" : {
+ "_index" : "index",
+ "_source" : {
+ "field3" : "_value3",
+ "field2" : "_value2",
+ "foo" : "rab"
+ },
+ "_ingest" : {
+ "pipeline" : "_simulate_pipeline",
+ "timestamp" : "2020-07-30T01:21:24.251863Z"
+ }
+ }
+ }
+ ]
+ }
+ ]
 }
 ----
-// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
 
 ////
 [source,console]

diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml
@@ -559,11 +559,15 @@ teardown:
  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
  - match: { docs.0.processor_results.0.doc._source.status: 200 }
+ - match: { docs.0.processor_results.0.status: "success" }
+ - match: { docs.0.processor_results.0.processor_type: "set" }
  - match: { docs.0.processor_results.1.tag: "rename-1" }
  - match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" }
  - match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" }
  - match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
  - match: { docs.0.processor_results.1.doc._source.status: 200 }
+ - match: { docs.0.processor_results.1.status: "error_ignored" }
+ - match: { docs.0.processor_results.1.processor_type: "rename" }
 
 ---
 "Test verbose simulate with ignore_failure and no exception thrown":
@@ -605,11 +609,16 @@ teardown:
  }
  - length: { docs: 1 }
  - length: { docs.0.processor_results: 2 }
+ - length: { docs.0.processor_results.0: 4 }
  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+ - match: { docs.0.processor_results.0.status: "success" }
+ - match: { docs.0.processor_results.0.processor_type: "set" }
  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
  - match: { docs.0.processor_results.0.doc._source.status: 200 }
- - length: { docs.0.processor_results.1: 2 }
+ - length: { docs.0.processor_results.1: 4 }
  - match: { docs.0.processor_results.1.tag: "rename-1" }
+ - match: { docs.0.processor_results.1.status: "success" }
+ - match: { docs.0.processor_results.1.processor_type: "rename" }
  - match: { docs.0.processor_results.1.doc._source.new_status: 200 }
 
 ---
@@ -725,7 +734,8 @@ teardown:
  {
  "set": {
  "field": "pipeline0",
- "value": true
+ "value": true,
+ "description" : "first_set"
  }
  },
  {
@@ -747,16 +757,25 @@ teardown:
  ]
  }
 - length: { docs: 1 }
-- length: { docs.0.processor_results: 3 }
+- length: { docs.0.processor_results: 5 }
 - match: { docs.0.processor_results.0.doc._source.pipeline0: true }
+- match: { docs.0.processor_results.0.status: "success" }
+- match: { docs.0.processor_results.0.processor_type: "set" }
+- match: { docs.0.processor_results.0.description: "first_set" }
 - is_false: docs.0.processor_results.0.doc._source.pipeline1
 - is_false: docs.0.processor_results.0.doc._source.pipeline2
-- match: { docs.0.processor_results.1.doc._source.pipeline0: true }
-- match: { docs.0.processor_results.1.doc._source.pipeline1: true }
-- is_false: docs.0.processor_results.1.doc._source.pipeline2
+- match: { docs.0.processor_results.1.doc: null }
+- match: { docs.0.processor_results.1.status: "success" }
+- match: { docs.0.processor_results.1.processor_type: "pipeline" }
 - match: { docs.0.processor_results.2.doc._source.pipeline0: true }
 - match: { docs.0.processor_results.2.doc._source.pipeline1: true }
-- match: { docs.0.processor_results.2.doc._source.pipeline2: true }
+- is_false: docs.0.processor_results.2.doc._source.pipeline2
+- match: { docs.0.processor_results.3.doc: null }
+- match: { docs.0.processor_results.3.status: "success" }
+- match: { docs.0.processor_results.3.processor_type: "pipeline" }
+- match: { docs.0.processor_results.4.doc._source.pipeline0: true }
+- match: { docs.0.processor_results.4.doc._source.pipeline1: true }
+- match: { docs.0.processor_results.4.doc._source.pipeline2: true }
 
 ---
 "Test verbose simulate with true conditional and on failure":
@@ -817,19 +836,27 @@ teardown:
 - length: { docs.0.processor_results: 4 }
 - match: { docs.0.processor_results.0.tag: "gunna_fail" }
 - match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" }
+- match: { docs.0.processor_results.0.status: "error" }
+- match: { docs.0.processor_results.0.processor_type: "rename" }
 - match: { docs.0.processor_results.1.tag: "failed1" }
 - match: { docs.0.processor_results.1.doc._source.failed1: "failed1" }
 - match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" }
+- match: { docs.0.processor_results.1.status: "success" }
+- match: { docs.0.processor_results.1.processor_type: "set" }
 - match: { docs.0.processor_results.2.tag: "gunna_fail_again" }
 - match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" }
+- match: { docs.0.processor_results.2.status: "error" }
+- match: { docs.0.processor_results.2.processor_type: "rename" }
 - match: { docs.0.processor_results.3.tag: "failed2" }
 - match: { docs.0.processor_results.3.doc._source.failed1: "failed1" }
 - match: { docs.0.processor_results.3.doc._source.failed2: "failed2" }
 - match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" }
+- match: { docs.0.processor_results.3.status: "success" }
+- match: { docs.0.processor_results.3.processor_type: "set" }
 
 
 ---
-"Test simulate with provided pipeline definition with tag and description in processors":
+"Test simulate with pipeline with conditional and skipped and dropped":
  - do:
  ingest.simulate:
  verbose: true
@@ -845,6 +872,16 @@ teardown:
  "field" : "field2",
  "value" : "_value"
  }
+ },
+ {
+ "drop" : {
+ "if": "false"
+ }
+ },
+ {
+ "drop" : {
+ "if": "true"
+ }
  }
  ]
  },
@@ -859,7 +896,43 @@ teardown:
  ]
  }
  - length: { docs: 1 }
- - length: { docs.0.processor_results: 1 }
+ - length: { docs.0.processor_results: 3 }
  - match: { docs.0.processor_results.0.doc._source.field2: "_value" }
  - match: { docs.0.processor_results.0.description: "processor_description" }
  - match: { docs.0.processor_results.0.tag: "processor_tag" }
+ - match: { docs.0.processor_results.0.status: "success" }
+ - match: { docs.0.processor_results.0.processor_type: "set" }
+ - match: { docs.0.processor_results.1.status: "skipped" }
+ - match: { docs.0.processor_results.1.processor_type: "drop" }
+ - match: { docs.0.processor_results.1.if.condition: "false" }
+ - match: { docs.0.processor_results.1.if.result: false }
+ - match: { docs.0.processor_results.2.status: "dropped" }
+ - match: { docs.0.processor_results.2.processor_type: "drop" }
+ - match: { docs.0.processor_results.2.if.condition: "true" }
+ - match: { docs.0.processor_results.2.if.result: true }
+---
+"Test simulate with provided pipeline that does not exist":
+ - do:
+ catch: bad_request
+ ingest.simulate:
+ verbose: true
+ body: >
+ {
+ "pipeline": {
+ "description": "_description",
+ "processors": [
+ {
+ "pipeline": {
+ "name": "____pipeline_doesnot_exist___"
+ }
+ }
+ ]
+ },
+ "docs": [
+ {
+ "_source": {}
+ }
+ ]
+ }
+ - match: { error.root_cause.0.type: "illegal_argument_exception" }
+ - match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" }