From 62b48d1854545cad7ee1f3a55d365cc257810262 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 18 Oct 2017 17:08:19 -0700 Subject: [PATCH] Add support for decimal timestamps to ts-epoch modifier --- plugins/inputs/logparser/README.md | 15 ++++- plugins/inputs/logparser/grok/grok.go | 26 ++++++-- plugins/inputs/logparser/grok/grok_test.go | 71 ++++++++++++++++++++++ 3 files changed, 107 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/logparser/README.md b/plugins/inputs/logparser/README.md index f823cfe68a541..2febb81947596 100644 --- a/plugins/inputs/logparser/README.md +++ b/plugins/inputs/logparser/README.md @@ -100,7 +100,7 @@ current time. - ts-rfc3339 ("2006-01-02T15:04:05Z07:00") - ts-rfc3339nano ("2006-01-02T15:04:05.999999999Z07:00") - ts-httpd ("02/Jan/2006:15:04:05 -0700") - - ts-epoch (seconds since unix epoch) + - ts-epoch (seconds since unix epoch, may contain decimal) - ts-epochnano (nanoseconds since unix epoch) - ts-"CUSTOM" @@ -130,6 +130,19 @@ This example input and config parses a file using a custom timestamp conversion: patterns = ['%{TIMESTAMP_ISO8601:timestamp:ts-"2006-01-02 15:04:05"} value=%{NUMBER:value:int}'] ``` +This example input and config parses a file using a timestamp in unix time: + +``` +1466004605 value=42 +1466004605.123456789 value=42 +``` + +```toml +[[inputs.logparser]] + [inputs.logparser.grok] + patterns = ['%{NUMBER:timestamp:ts-epoch} value=%{NUMBER:value:int}'] +``` + This example parses a file using a built-in conversion and a custom pattern: ``` diff --git a/plugins/inputs/logparser/grok/grok.go b/plugins/inputs/logparser/grok/grok.go index 0be18b54c767c..491a1374818e1 100644 --- a/plugins/inputs/logparser/grok/grok.go +++ b/plugins/inputs/logparser/grok/grok.go @@ -253,12 +253,30 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) { case STRING: fields[k] = strings.Trim(v, `"`) case EPOCH: - iv, err := strconv.ParseInt(v, 10, 64) + parts := strings.SplitN(v, ".", 2) + if len(parts) == 0 { + log.Printf("E! Error parsing %s to timestamp: %s", v, err) + break + } + + sec, err := strconv.ParseInt(parts[0], 10, 64) if err != nil { - log.Printf("E! Error parsing %s to int: %s", v, err) - } else { - timestamp = time.Unix(iv, 0) + log.Printf("E! Error parsing %s to timestamp: %s", v, err) + break + } + ts := time.Unix(sec, 0) + + if len(parts) == 2 { + padded := fmt.Sprintf("%-9s", parts[1]) + nsString := strings.Replace(padded[:9], " ", "0", -1) + nanosec, err := strconv.ParseInt(nsString, 10, 64) + if err != nil { + log.Printf("E! Error parsing %s to timestamp: %s", v, err) + break + } + ts = ts.Add(time.Duration(nanosec) * time.Nanosecond) } + timestamp = ts case EPOCH_NANO: iv, err := strconv.ParseInt(v, 10, 64) if err != nil { diff --git a/plugins/inputs/logparser/grok/grok_test.go b/plugins/inputs/logparser/grok/grok_test.go index 6d07b6ecaf72a..480502d6c2170 100644 --- a/plugins/inputs/logparser/grok/grok_test.go +++ b/plugins/inputs/logparser/grok/grok_test.go @@ -385,6 +385,77 @@ func TestParseEpoch(t *testing.T) { assert.Equal(t, time.Unix(1466004605, 0), metricA.Time()) } +func TestParseEpochDecimal(t *testing.T) { + var tests = []struct { + name string + line string + noMatch bool + err error + tags map[string]string + fields map[string]interface{} + time time.Time + }{ + { + name: "ns precision", + line: "1466004605.359052000 value=42", + tags: map[string]string{}, + fields: map[string]interface{}{ + "value": int64(42), + }, + time: time.Unix(0, 1466004605359052000), + }, + { + name: "ms precision", + line: "1466004605.359 value=42", + tags: map[string]string{}, + fields: map[string]interface{}{ + "value": int64(42), + }, + time: time.Unix(0, 1466004605359000000), + }, + { + name: "second precision", + line: "1466004605 value=42", + tags: map[string]string{}, + fields: map[string]interface{}{ + "value": int64(42), + }, + time: time.Unix(0, 1466004605000000000), + }, + { + name: "sub ns precision", + line: "1466004605.123456789123 value=42", + tags: map[string]string{}, + fields: map[string]interface{}{ + "value": int64(42), + }, + time: time.Unix(0, 1466004605123456789), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := &Parser{ + Patterns: []string{"%{NUMBER:ts:ts-epoch} value=%{NUMBER:value:int}"}, + } + assert.NoError(t, parser.Compile()) + m, err := parser.ParseLine(tt.line) + + if tt.noMatch { + require.Nil(t, m) + require.Nil(t, err) + return + } + + require.Equal(t, tt.err, err) + + require.NotNil(t, m) + require.Equal(t, tt.tags, m.Tags()) + require.Equal(t, tt.fields, m.Fields()) + require.Equal(t, tt.time, m.Time()) + }) + } +} + func TestParseEpochErrors(t *testing.T) { p := &Parser{ Patterns: []string{"%{MYAPP}"},