Configurable output format for date processor (#61324)

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java

@@ -43,17 +43,24 @@ public final class DateProcessor extends AbstractProcessor {
 
  public static final String TYPE = "date";
  static final String DEFAULT_TARGET_FIELD = "@timestamp";
- private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+ static final String DEFAULT_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
 
+ private final DateFormatter formatter;
  private final TemplateScript.Factory timezone;
  private final TemplateScript.Factory locale;
  private final String field;
  private final String targetField;
  private final List<String> formats;
  private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
+ private final String outputFormat;
 
  DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
  String field, List<String> formats, String targetField) {
+ this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT);
+ }
+
+ DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
+ String field, List<String> formats, String targetField, String outputFormat) {
  super(tag, description);
  this.timezone = timezone;
  this.locale = locale;
@@ -65,6 +72,8 @@ public final class DateProcessor extends AbstractProcessor {
  DateFormat dateFormat = DateFormat.fromString(format);
  dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
  }
+ this.outputFormat = outputFormat;
+ formatter = DateFormatter.forPattern(this.outputFormat);
  }
 
  private ZoneId newDateTimeZone(Map<String, Object> params) {
@@ -99,7 +108,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
  throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
  }
 
- ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime));
+ ingestDocument.setFieldValue(targetField, formatter.format(dateTime));
  return ingestDocument;
  }
 
@@ -128,6 +137,10 @@ List<String> getFormats() {
  return formats;
  }
 
+ String getOutputFormat() {
+ return outputFormat;
+ }
+
  public static final class Factory implements Processor.Factory {
 
  private final ScriptService scriptService;
@@ -153,8 +166,16 @@ public DateProcessor create(Map<String, Processor.Factory> registry, String proc
  "locale", localeString, scriptService);
  }
  List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
+ String outputFormat =
+ ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
+ try {
+ DateFormatter.forPattern(outputFormat);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e);
+ }
+
  return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
- targetField);
+ targetField, outputFormat);
  }
  }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java

@@ -146,4 +146,42 @@ public void testParseTargetField() throws Exception {
  DateProcessor processor = factory.create(null, null, null, config);
  assertThat(processor.getTargetField(), equalTo(targetField));
  }
+
+ public void testParseOutputFormat() throws Exception {
+ final String outputFormat = "dd:MM:yyyy";
+ Map<String, Object> config = new HashMap<>();
+ String sourceField = randomAlphaOfLengthBetween(1, 10);
+ String targetField = randomAlphaOfLengthBetween(1, 10);
+ config.put("field", sourceField);
+ config.put("target_field", targetField);
+ config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+ config.put("output_format", outputFormat);
+ DateProcessor processor = factory.create(null, null, null, config);
+ assertThat(processor.getOutputFormat(), equalTo(outputFormat));
+ }
+
+ public void testDefaultOutputFormat() throws Exception {
+ Map<String, Object> config = new HashMap<>();
+ String sourceField = randomAlphaOfLengthBetween(1, 10);
+ String targetField = randomAlphaOfLengthBetween(1, 10);
+ config.put("field", sourceField);
+ config.put("target_field", targetField);
+ config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+ DateProcessor processor = factory.create(null, null, null, config);
+ assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
+ }
+
+ public void testInvalidOutputFormatRejected() throws Exception {
+ final String outputFormat = "invalid_date_format";
+ Map<String, Object> config = new HashMap<>();
+ String sourceField = randomAlphaOfLengthBetween(1, 10);
+ String targetField = randomAlphaOfLengthBetween(1, 10);
+ config.put("field", sourceField);
+ config.put("target_field", targetField);
+ config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+ config.put("output_format", outputFormat);
+
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
+ assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
+ }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
 
+import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -224,4 +225,17 @@ null, templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Fac
  assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
  assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid"));
  }
+
+ public void testOutputFormat() {
+ long nanosAfterEpoch = randomLongBetween(1, 999999);
+ DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), null, null, null,
+ "date_as_string", Collections.singletonList("iso8601"), "date_as_date", "HH:mm:ss.SSSSSSSSS");
+ Map<String, Object> document = new HashMap<>();
+ document.put("date_as_string", Instant.EPOCH.plusNanos(nanosAfterEpoch).toString());
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ processor.execute(ingestDocument);
+ // output format is time only with nanosecond precision
+ String expectedDate = "00:00:00." + String.format(Locale.ROOT, "%09d", nanosAfterEpoch);
+ assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate));
+ }
 }