From 05aa3c79ec92abc5671a5acf284c6cbef03d8bf8 Mon Sep 17 00:00:00 2001 From: Henri DF Date: Tue, 29 Sep 2020 17:22:24 +0200 Subject: [PATCH 1/2] Add support for timestamps with +- tz offsets Suricata outputs timestamp like 2015-04-13T11:32:45.143323+0200, whereas Zeek does 2015-04-13T11:32:45.143323Z02:00. This commit lets us parse both. --- zio/ndjsonio/ndjson_test.go | 6 ++++++ zio/ndjsonio/typeparser.go | 28 +++++++++++++++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/zio/ndjsonio/ndjson_test.go b/zio/ndjsonio/ndjson_test.go index 25dd1079f6..d9d291a6ab 100644 --- a/zio/ndjsonio/ndjson_test.go +++ b/zio/ndjsonio/ndjson_test.go @@ -193,6 +193,12 @@ func TestNewRawFromJSON(t *testing.T) { 0:[test;-;-;-;1573860644.637486;-;]`, json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486Z"}`, }, + { + name: "TsISO8601-0700", + tzng: `#0:record[_path:string,b:bool,i:int64,s:set[bool],ts:time,v:array[int64]] +0:[test;-;-;-;1573864244.637486;-;]`, + json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486-0100"}`, + }, { name: "TsEpoch", tzng: `#0:record[_path:string,ts:time] diff --git a/zio/ndjsonio/typeparser.go b/zio/ndjsonio/typeparser.go index 53607dfe39..87df37b000 100644 --- a/zio/ndjsonio/typeparser.go +++ b/zio/ndjsonio/typeparser.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/brimsec/zq/pkg/byteconv" "github.com/brimsec/zq/pkg/nano" @@ -358,22 +359,35 @@ func parseSimpleType(value []byte, typ zng.Type) ([]byte, error) { } } +func parseISO8601SignedOffset(s []byte) (nano.Ts, error) { + t, err := time.Parse("2006-01-02T15:04:05.999999999-0700", string(s)) + if err != nil { + return 0, err + } + return nano.TimeToTs(t), nil +} + // parseJSONTimestamp interprets data as a timestamp and returns its value as // both a nano.Ts and the standard Zeek format (a decimal floating-point number // representing seconds since the Unix epoch). // -// parseJSONTimestamp understands the three timestamp formats that Zeek's ASCII -// log writer can produce when LogAscii::use_json is true. These formats -// correspond to the three possible values for LogAscii::json_timestamps: -// JSON::TS_EPOCH, JSON::TS_ISO8601, and JSON::TS_MILLIS. For descriptions, see +// parseJSONTimestamp understands the three timestamp formats that +// Zeek's ASCII log writer can produce when LogAscii::use_json is true +// as well as the ISO8601 format emitted by Suricata in eve.json. +// +// The Zeek formats correspond to the three possible values for +// LogAscii::json_timestamps: JSON::TS_EPOCH, JSON::TS_ISO8601, and +// JSON::TS_MILLIS. For descriptions, see // https://docs.zeek.org/en/stable/scripts/base/init-bare.zeek.html#type-JSON::TimestampFormat. func parseJSONTimestamp(data []byte) (nano.Ts, error) { switch { - case bytes.Contains(data, []byte{'-'}): // JSON::TS_ISO8601 + case bytes.Contains(data, []byte{'Z'}): // Zeek JSON::TS_ISO8601 return nano.ParseRFC3339Nano(data) - case bytes.Contains(data, []byte{'.'}): // JSON::TS_EPOCH + case bytes.Contains(data, []byte{'-'}): + return parseISO8601SignedOffset(data) + case bytes.Contains(data, []byte{'.'}): // Zeek JSON::TS_EPOCH return nano.Parse(data) - default: // JSON::TS_MILLIS + default: // Zeek JSON::TS_MILLIS return nano.ParseMillis(data) } } From deef6217c7435bc0c2bc9c641e1a5ef1ab1e28a7 Mon Sep 17 00:00:00 2001 From: Henri DF Date: Thu, 1 Oct 2020 21:50:06 +0200 Subject: [PATCH 2/2] Update zio/ndjsonio/ndjson_test.go --- zio/ndjsonio/ndjson_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio/ndjsonio/ndjson_test.go b/zio/ndjsonio/ndjson_test.go index d9d291a6ab..bc4223ca5b 100644 --- a/zio/ndjsonio/ndjson_test.go +++ b/zio/ndjsonio/ndjson_test.go @@ -194,7 +194,7 @@ func TestNewRawFromJSON(t *testing.T) { json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486Z"}`, }, { - name: "TsISO8601-0700", + name: "TsISO8601-0100", tzng: `#0:record[_path:string,b:bool,i:int64,s:set[bool],ts:time,v:array[int64]] 0:[test;-;-;-;1573864244.637486;-;]`, json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486-0100"}`,