Per processor description for verbose simulate (elastic#58207)

For ingest node processors a per processor description
was recently added. This commit displays that description
in the verbose output of the pipeline simulation.

related elastic#57906

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml

@@ -829,7 +829,7 @@ teardown:
 
 
 ---
-"Test simulate with provided pipeline definition with description in processors":
+"Test simulate with provided pipeline definition with tag and description in processors":
  - do:
  ingest.simulate:
  verbose: true
@@ -841,6 +841,7 @@ teardown:
  {
  "set" : {
  "description": "processor_description",
+ "tag": "processor_tag",
  "field" : "field2",
  "value" : "_value"
  }
@@ -860,3 +861,5 @@ teardown:
  - length: { docs: 1 }
  - length: { docs.0.processor_results: 1 }
  - match: { docs.0.processor_results.0.doc._source.field2: "_value" }
+ - match: { docs.0.processor_results.0.description: "processor_description" }
+ - match: { docs.0.processor_results.0.tag: "processor_tag" }

server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -39,6 +40,7 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject {
 
  private static final String IGNORED_ERROR_FIELD = "ignored_error";
  private final String processorTag;
+ private final String description;
  private final WriteableIngestDocument ingestDocument;
  private final Exception failure;
 
@@ -62,18 +64,20 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject {
  true,
  a -> {
  String processorTag = a[0] == null ? null : (String)a[0];
- IngestDocument document = a[1] == null ? null : ((WriteableIngestDocument)a[1]).getIngestDocument();
+ String description = a[1] == null ? null : (String)a[1];
+ IngestDocument document = a[2] == null ? null : ((WriteableIngestDocument)a[2]).getIngestDocument();
  Exception failure = null;
- if (a[2] != null) {
- failure = (ElasticsearchException)a[2];
- } else if (a[3] != null) {
+ if (a[3] != null) {
  failure = (ElasticsearchException)a[3];
+ } else if (a[4] != null) {
+ failure = (ElasticsearchException)a[4];
  }
- return new SimulateProcessorResult(processorTag, document, failure);
+ return new SimulateProcessorResult(processorTag, description, document, failure);
  }
  );
  static {
  PARSER.declareString(optionalConstructorArg(), new ParseField(ConfigurationUtils.TAG_KEY));
+ PARSER.declareString(optionalConstructorArg(), new ParseField(ConfigurationUtils.DESCRIPTION_KEY));
  PARSER.declareObject(
  optionalConstructorArg(),
  WriteableIngestDocument.INGEST_DOC_PARSER,
@@ -91,22 +95,24 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject {
  );
  }
 
- public SimulateProcessorResult(String processorTag, IngestDocument ingestDocument, Exception failure) {
+ public SimulateProcessorResult(String processorTag, String description, IngestDocument ingestDocument,
+ Exception failure) {
  this.processorTag = processorTag;
+ this.description = description;
  this.ingestDocument = (ingestDocument == null) ? null : new WriteableIngestDocument(ingestDocument);
  this.failure = failure;
  }
 
- public SimulateProcessorResult(String processorTag, IngestDocument ingestDocument) {
- this(processorTag, ingestDocument, null);
+ public SimulateProcessorResult(String processorTag, String description, IngestDocument ingestDocument) {
+ this(processorTag, description, ingestDocument, null);
  }
 
- public SimulateProcessorResult(String processorTag, Exception failure) {
- this(processorTag, null, failure);
+ public SimulateProcessorResult(String processorTag, String description, Exception failure) {
+ this(processorTag, description, null, failure);
  }
 
- public SimulateProcessorResult(String processorTag) {
- this(processorTag, null, null);
+ public SimulateProcessorResult(String processorTag, String description) {
+ this(processorTag, description, null, null);
  }
 
  /**
@@ -116,13 +122,21 @@ public SimulateProcessorResult(String processorTag) {
  this.processorTag = in.readString();
  this.ingestDocument = in.readOptionalWriteable(WriteableIngestDocument::new);
  this.failure = in.readException();
+ if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+ this.description = in.readOptionalString();
+ } else {
+ this.description = null;
+ }
  }
 
  @Override
  public void writeTo(StreamOutput out) throws IOException {
  out.writeString(processorTag);
  out.writeOptionalWriteable(ingestDocument);
  out.writeException(failure);
+ if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+ out.writeOptionalString(description);
+ }
  }
 
  public IngestDocument getIngestDocument() {
@@ -140,6 +154,10 @@ public Exception getFailure() {
  return failure;
  }
 
+ public String getDescription() {
+ return description;
+ }
+
  @Override
  public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
  if (processorTag == null && failure == null && ingestDocument == null) {
@@ -153,6 +171,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  builder.field(ConfigurationUtils.TAG_KEY, processorTag);
  }
 
+ if (description != null) {
+ builder.field(ConfigurationUtils.DESCRIPTION_KEY, description);
+ }
+
  if (failure != null && ingestDocument != null) {
  builder.startObject(IGNORED_ERROR_FIELD);
  ElasticsearchException.generateFailureXContent(builder, params, failure, true);

server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java

@@ -66,9 +66,10 @@ public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Ex
  if (elasticsearchException.getCause() instanceof IllegalStateException) {
  if (ignoreFailure) {
  processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getTag(),
- new IngestDocument(ingestDocument), e));
+ pipelineProcessor.getDescription(), new IngestDocument(ingestDocument), e));
  } else {
- processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getTag(), e));
+ processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getTag(),
+ pipelineProcessor.getDescription(), e));
  }
  handler.accept(null, elasticsearchException);
  }
@@ -86,17 +87,21 @@ public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Ex
  actualProcessor.execute(ingestDocument, (result, e) -> {
  if (e != null) {
  if (ignoreFailure) {
- processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument), e));
+ processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), new IngestDocument(ingestDocument), e));
  } else {
- processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), e));
+ processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), e));
  }
  handler.accept(null, e);
  } else {
  if (result != null) {
- processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument)));
+ processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), new IngestDocument(ingestDocument)));
  handler.accept(result, null);
  } else {
- processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag()));
+ processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription()));
  handler.accept(null, null);
  }
  }

server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java

@@ -19,11 +19,13 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.StringJoiner;
@@ -50,6 +52,7 @@ public void testSerialization() throws IOException {
  StreamInput streamInput = out.bytes().streamInput();
  SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(streamInput);
  assertThat(otherSimulateProcessorResult.getProcessorTag(), equalTo(simulateProcessorResult.getProcessorTag()));
+ assertThat(otherSimulateProcessorResult.getDescription(), equalTo(simulateProcessorResult.getDescription()));
  if (isSuccessful) {
  assertIngestDocument(otherSimulateProcessorResult.getIngestDocument(), simulateProcessorResult.getIngestDocument());
  if (isIgnoredException) {
@@ -67,19 +70,36 @@ public void testSerialization() throws IOException {
  }
  }
 
+ public void testBWCDescription() throws IOException {
+ boolean isSuccessful = randomBoolean();
+ boolean isIgnoredException = randomBoolean();
+ SimulateProcessorResult simulateProcessorResult = createTestInstance(isSuccessful, isIgnoredException);
+
+ BytesStreamOutput out = new BytesStreamOutput();
+ out.setVersion(VersionUtils.getPreviousVersion(Version.V_7_9_0));
+ simulateProcessorResult.writeTo(out);
+ StreamInput in = out.bytes().streamInput();
+ in.setVersion(VersionUtils.getPreviousVersion(Version.V_7_9_0));
+ SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(in);
+ assertNull(otherSimulateProcessorResult.getDescription());
+ }
+
  static SimulateProcessorResult createTestInstance(boolean isSuccessful,
  boolean isIgnoredException) {
  String processorTag = randomAlphaOfLengthBetween(1, 10);
+ String description = randomAlphaOfLengthBetween(1, 10);
  SimulateProcessorResult simulateProcessorResult;
  if (isSuccessful) {
  IngestDocument ingestDocument = createRandomIngestDoc();
  if (isIgnoredException) {
- simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test"));
+ simulateProcessorResult = new SimulateProcessorResult(processorTag, description, ingestDocument,
+ new IllegalArgumentException("test"));
  } else {
- simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument);
+ simulateProcessorResult = new SimulateProcessorResult(processorTag, description, ingestDocument);
  }
  } else {
- simulateProcessorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test"));
+ simulateProcessorResult = new SimulateProcessorResult(processorTag, description,
+ new IllegalArgumentException("test"));
  }
  return simulateProcessorResult;
  }

server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java

@@ -67,7 +67,8 @@ public void testActualProcessor() throws Exception {
  TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(false, actualProcessor, null, resultList);
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
 
  assertThat(actualProcessor.getInvokedCounter(), equalTo(1));
  assertThat(resultList.size(), equalTo(1));
@@ -87,7 +88,8 @@ public void testActualCompoundProcessorWithoutOnFailure() throws Exception {
  trackingProcessor.execute(ingestDocument, (result, e) -> holder[0] = e);
  assertThat(((IngestProcessorException) holder[0]).getRootCause().getMessage(), equalTo(exception.getMessage()));
 
- SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  assertThat(testProcessor.getInvokedCounter(), equalTo(1));
  assertThat(resultList.size(), equalTo(1));
  assertThat(resultList.get(0).getIngestDocument(), nullValue());
@@ -107,8 +109,10 @@ public void testActualCompoundProcessorWithOnFailure() throws Exception {
  CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), ingestDocument);
- SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(),
+ failProcessor.getDescription(), ingestDocument);
+ SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(),
+ failProcessor.getDescription(), ingestDocument);
 
  assertThat(failProcessor.getInvokedCounter(), equalTo(2));
  assertThat(onFailureProcessor.getInvokedCounter(), equalTo(2));
@@ -159,8 +163,10 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex
  trackingProcessor.execute(ingestDocument, (result, e) -> {
  });
 
- SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), ingestDocument);
- SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(),
+ failProcessor.getDescription(), ingestDocument);
+ SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(),
+ onFailureProcessor.getDescription(), ingestDocument);
 
  assertThat(failProcessor.getInvokedCounter(), equalTo(1));
  assertThat(onFailureProcessor.getInvokedCounter(), equalTo(1));
@@ -188,7 +194,8 @@ public void testActualCompoundProcessorWithIgnoreFailure() throws Exception {
 
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(testProcessor.getTag(),
+ testProcessor.getDescription(), ingestDocument);
  assertThat(testProcessor.getInvokedCounter(), equalTo(1));
  assertThat(resultList.size(), equalTo(1));
  assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
@@ -218,7 +225,8 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception {
 
  CompoundProcessor trackingProcessor = decorate(compoundProcessor, null, resultList);
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(compoundProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(compoundProcessor.getTag(),
+ compoundProcessor.getDescription(), ingestDocument);
 
  //the step for key 2 is never executed due to conditional and thus not part of the result set
  assertThat(resultList.size(), equalTo(2));
@@ -262,7 +270,8 @@ pipelineId, null, null, new CompoundProcessor(
 
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId);
 
  verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId);
@@ -331,7 +340,8 @@ pipelineId2, null, null, new CompoundProcessor(
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId1);
 
  verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId1);
@@ -401,7 +411,8 @@ pipelineId2, null, null, new CompoundProcessor(
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId1);
 
  verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId1);
@@ -453,7 +464,8 @@ pipelineId, null, null, new CompoundProcessor(
 
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId);
 
  verify(ingestService, Mockito.atLeast(2)).getPipeline(pipelineId);
@@ -530,7 +542,8 @@ pipelineId, null, null, new CompoundProcessor(
 
  trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
- SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+ SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(),
+ actualProcessor.getDescription(), ingestDocument);
  expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId);
 
  verify(ingestService, Mockito.atLeast(2)).getPipeline(pipelineId);