diff --git a/api/api.go b/api/api.go index 9e513621c3..1c2803a76b 100644 --- a/api/api.go +++ b/api/api.go @@ -12,7 +12,6 @@ import ( "github.com/brimdata/zed/pkg/iosrc" "github.com/brimdata/zed/pkg/nano" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/zjsonio" ) @@ -162,10 +161,9 @@ type PcapPostStatus struct { } type LogPostRequest struct { - Paths []string `json:"paths"` - StopErr bool `json:"stop_err"` - JSONTypeConfig *ndjsonio.TypeConfig `json:"json_type_config"` - Shaper json.RawMessage `json:"shaper,omitempty"` + Paths []string `json:"paths"` + StopErr bool `json:"stop_err"` + Shaper json.RawMessage `json:"shaper,omitempty"` } type LogPostWarning struct { diff --git a/api/client/connection.go b/api/client/connection.go index 9af338a0b8..9d6928401f 100644 --- a/api/client/connection.go +++ b/api/client/connection.go @@ -16,7 +16,6 @@ import ( "github.com/brimdata/zed/api" "github.com/brimdata/zed/compiler/ast" "github.com/brimdata/zed/pcap/pcapio" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zqe" "github.com/go-resty/resty/v2" ) @@ -398,7 +397,6 @@ type PcapReadCloser struct { } type LogPostOpts struct { - JSON *ndjsonio.TypeConfig StopError bool Shaper ast.Proc } @@ -417,9 +415,8 @@ func (c *Connection) LogPostPath(ctx context.Context, space api.SpaceID, opts *L func (c *Connection) LogPostPathStream(ctx context.Context, space api.SpaceID, opts *LogPostOpts, paths ...string) (*Stream, error) { body := api.LogPostRequest{ - Paths: paths, - JSONTypeConfig: opts.JSON, - StopErr: opts.StopError, + Paths: paths, + StopErr: opts.StopError, } if opts != nil && opts.Shaper != nil { raw, err := json.Marshal(opts.Shaper) @@ -468,9 +465,6 @@ func (c *Connection) LogPostWriter(ctx context.Context, space api.SpaceID, opts if opts.StopError { req.SetQueryParam("stop_err", "true") } - if opts.JSON != nil { - writer.SetJSONConfig(opts.JSON) - } } u := path.Join("/space", url.PathEscape(string(space)), "log") resp, err := req.Post(u) diff --git a/api/client/multipartwriter.go b/api/client/multipartwriter.go index 5a6b325b02..9b9ef42c7e 100644 --- a/api/client/multipartwriter.go +++ b/api/client/multipartwriter.go @@ -12,7 +12,6 @@ import ( "github.com/brimdata/zed/compiler/ast" "github.com/brimdata/zed/pkg/iosrc" - "github.com/brimdata/zed/zio/ndjsonio" ) type MultipartWriter struct { @@ -25,7 +24,6 @@ type MultipartWriter struct { start sync.Once readers []io.Reader uris []iosrc.URI - json *ndjsonio.TypeConfig shaper ast.Proc } @@ -60,10 +58,6 @@ func MultipartDataWriter(readers ...io.Reader) (*MultipartWriter, error) { return m, nil } -func (m *MultipartWriter) SetJSONConfig(config *ndjsonio.TypeConfig) { - m.json = config -} - func (m *MultipartWriter) SetShaper(shaper ast.Proc) { m.shaper = shaper } @@ -84,10 +78,6 @@ func (m *MultipartWriter) Read(b []byte) (int, error) { } func (m *MultipartWriter) run() { - if err := m.sendJSONConfig(); err != nil { - m.pw.CloseWithError(err) - return - } if err := m.sendShaperAST(); err != nil { m.pw.CloseWithError(err) return @@ -126,17 +116,6 @@ func (m *MultipartWriter) write(name string, r io.Reader) error { return err } -func (m *MultipartWriter) sendJSONConfig() error { - if m.json == nil { - return nil - } - w, err := m.form.CreateFormField("json_config") - if err != nil { - return err - } - return json.NewEncoder(w).Encode(m.json) -} - func (m *MultipartWriter) sendShaperAST() error { if m.shaper == nil { return nil diff --git a/cli/inputflags/flags.go b/cli/inputflags/flags.go index be12812195..c445367df7 100644 --- a/cli/inputflags/flags.go +++ b/cli/inputflags/flags.go @@ -1,20 +1,16 @@ package inputflags import ( - "encoding/json" "errors" "flag" "fmt" - "io/ioutil" "os" - "regexp" "github.com/brimdata/zed/cli/auto" "github.com/brimdata/zed/pkg/iosrc" "github.com/brimdata/zed/zbuf" "github.com/brimdata/zed/zio" "github.com/brimdata/zed/zio/detector" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/zngio" "github.com/brimdata/zed/zson" ) @@ -23,8 +19,6 @@ type Flags struct { zio.ReaderOpts ReadMax auto.Bytes ReadSize auto.Bytes - // The JSON type config is loaded from the types filie when Init is called. - jsonTypesFile string } func (f *Flags) Options() zio.ReaderOpts { @@ -34,9 +28,6 @@ func (f *Flags) Options() zio.ReaderOpts { func (f *Flags) SetFlags(fs *flag.FlagSet) { fs.StringVar(&f.Format, "i", "auto", "format of input data [auto,zng,zst,json,ndjson,zeek,zjson,csv,tzng,parquet]") fs.BoolVar(&f.Zng.Validate, "validate", true, "validate the input format when reading ZNG streams") - fs.StringVar(&f.jsonTypesFile, "j", "", "path to json types file") - fs.StringVar(&f.JSON.PathRegexp, "pathregexp", ndjsonio.DefaultPathRegexp, - "regexp for extracting _path from json log name (when -inferpath=true)") f.ReadMax = auto.NewBytes(zngio.MaxSize) fs.Var(&f.ReadMax, "readmax", "maximum memory used read buffers in MiB, MB, etc") f.ReadSize = auto.NewBytes(zngio.ReadSize) @@ -45,17 +36,6 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) { // Init is called after flags have been parsed. func (f *Flags) Init() error { - // Catch errors early ... ?! XXX - if _, err := regexp.Compile(f.JSON.PathRegexp); err != nil { - return err - } - if f.jsonTypesFile != "" { - c, err := LoadJSONConfig(f.jsonTypesFile) - if err != nil { - return err - } - f.JSON.TypeConfig = c - } f.Zng.Max = int(f.ReadMax.Bytes) if f.Zng.Max < 0 { return errors.New("max read buffer size must be greater than zero") @@ -67,21 +47,6 @@ func (f *Flags) Init() error { return nil } -func LoadJSONConfig(path string) (*ndjsonio.TypeConfig, error) { - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - var tc ndjsonio.TypeConfig - if err := json.Unmarshal(data, &tc); err != nil { - return nil, fmt.Errorf("%s: unmarshaling error: %s", path, err) - } - if err := tc.Validate(); err != nil { - return nil, fmt.Errorf("%s: %s", path, err) - } - return &tc, nil -} - func (f *Flags) Open(zctx *zson.Context, paths []string, stopOnErr bool) ([]zbuf.Reader, error) { var readers []zbuf.Reader for _, path := range paths { diff --git a/cmd/zsontypes/main.go b/cmd/zsontypes/main.go deleted file mode 100644 index ebdce96838..0000000000 --- a/cmd/zsontypes/main.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "fmt" - "os" -) - -func main() { - if _, err := ZsonTypes.ExecRoot(os.Args[1:]); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - os.Exit(1) - } -} diff --git a/cmd/zsontypes/zsontypes.go b/cmd/zsontypes/zsontypes.go deleted file mode 100644 index 720b6645b7..0000000000 --- a/cmd/zsontypes/zsontypes.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "flag" - "fmt" - - "github.com/brimdata/zed/cli/inputflags" - "github.com/brimdata/zed/pkg/charm" - "github.com/brimdata/zed/zio/ndjsonio/compat" - "github.com/brimdata/zed/zio/tzngio" - "github.com/brimdata/zed/zng" - "github.com/brimdata/zed/zson" -) - -var ZsonTypes = &charm.Spec{ - Name: "zsontypes", - Usage: "zsontypes ", - Short: "tool for printing zson types for legacy types.json", - Long: ` -The Zsontypes tool prints a ZSON-formatted string for each type defined in a legacy types.json file. -It is intended to help those transitioning from the "types.json" approach to the newer Z-based approach. - -`, - New: func(parent charm.Command, flags *flag.FlagSet) (charm.Command, error) { - return New(flags) - }, -} - -func init() { - ZsonTypes.Add(charm.Help) -} - -type Command struct{} - -func New(f *flag.FlagSet) (charm.Command, error) { - return &Command{}, nil -} - -func (c *Command) Run(args []string) error { - if len(args) != 1 { - return ZsonTypes.Exec(c, []string{"help"}) - } - fname := args[0] - return c.printTypes(fname) -} - -func (c *Command) printTypes(fname string) error { - tc, err := inputflags.LoadJSONConfig(fname) - if err != nil { - return err - } - zctx := zson.NewContext() - // (from ndjson.io.NewReader) - // Note: we add hardwired aliases for "port" to "uint16" when reading - // *any* json file but they are only used when the schema mapper - // (aka typings config) references such types from a configured schema. - // However, the schema mapper should be responsible for creating these - // aliases according to its configuration. See issue #1427. - if _, err := zctx.LookupTypeAlias("zenum", zng.TypeString); err != nil { - return err - } - if _, err = zctx.LookupTypeAlias("port", zng.TypeUint16); err != nil { - return err - } - tp := tzngio.NewTypeParser(zctx) - - for key, columns := range tc.Descriptors { - typeName, err := compat.DecodeType(columns) - if err != nil { - return fmt.Errorf("error decoding type \"%s\": %s", typeName, err) - } - typ, err := tp.Parse(typeName) - if err != nil { - return err - } - fmt.Printf("%s: %s\n", key, zson.FormatType(typ)) - } - return nil -} diff --git a/go.mod b/go.mod index 5b30bb9383..fd011a22a8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/araddon/dateparse v0.0.0-20210207001429-0eec95c9db7e github.com/aws/aws-sdk-go v1.36.17 github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f - github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/fraugster/parquet-go v0.3.0 github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index 49db4b852b..3796c508dc 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brimdata/parquet-go v0.3.1 h1:px/EuItj2/lWMJDcx9H412f4Fl1CNdZSHEtdvniOYq4= github.com/brimdata/parquet-go v0.3.1/go.mod h1:qIL8Wm6AK06QHCj9OBFW6PyS+7ukZxc20K/acSeGUas= -github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e h1:oJCXMss/3rg5F6Poy9wG3JQusc58Mzk5B9Z6wSnssNE= -github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e/go.mod h1:errmMKH8tTB49UR2A8C8DPYkyudelsYJwJFaZHQ6ik8= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/ppl/zqd/handlers.go b/ppl/zqd/handlers.go index a282fbf437..03d67425a9 100644 --- a/ppl/zqd/handlers.go +++ b/ppl/zqd/handlers.go @@ -657,7 +657,7 @@ func handleIntakePostData(c *Core, w http.ResponseWriter, r *http.Request) { return } zctx := zson.NewContext() - zr, err := detector.NewReaderWithOpts(r.Body, zctx, "", zio.ReaderOpts{Zng: zngio.ReaderOpts{Validate: true}}) + zr, err := detector.NewReaderWithOpts(r.Body, zctx, zio.ReaderOpts{Zng: zngio.ReaderOpts{Validate: true}}) if err != nil { respondError(c, w, r, err) return diff --git a/ppl/zqd/handlers_test.go b/ppl/zqd/handlers_test.go index f5684902b8..973f4d8c14 100644 --- a/ppl/zqd/handlers_test.go +++ b/ppl/zqd/handlers_test.go @@ -26,7 +26,6 @@ import ( "github.com/brimdata/zed/ppl/zqd" "github.com/brimdata/zed/zbuf" "github.com/brimdata/zed/zio" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/zsonio" "github.com/brimdata/zed/zson" "github.com/prometheus/client_golang/prometheus" @@ -468,29 +467,8 @@ detectablebutbadline` func TestPostNDJSONLogs(t *testing.T) { const src = `{"ts":"1000","uid":"CXY9a54W2dLZwzPXf1","_path":"http"} {"ts":"2000","uid":"CXY9a54W2dLZwzPXf1","_path":"http"}` - const expected = `{_path:"http",ts:1970-01-01T00:00:02Z,uid:"CXY9a54W2dLZwzPXf1" (bstring)} (=0) -{_path:"http",ts:1970-01-01T00:00:01Z,uid:"CXY9a54W2dLZwzPXf1"} (0)` - tc := ndjsonio.TypeConfig{ - Descriptors: map[string][]interface{}{ - "http_log": []interface{}{ - map[string]interface{}{ - "name": "_path", - "type": "string", - }, - map[string]interface{}{ - "name": "ts", - "type": "time", - }, - map[string]interface{}{ - "name": "uid", - "type": "bstring", - }, - }, - }, - Rules: []ndjsonio.Rule{ - ndjsonio.Rule{"_path", "http", "http_log"}, - }, - } + const expected = `{ts:"2000",uid:"CXY9a54W2dLZwzPXf1",_path:"http"} +{ts:"1000",uid:"CXY9a54W2dLZwzPXf1",_path:"http"}` test := func(input string) { _, conn := newCore(t) @@ -498,14 +476,12 @@ func TestPostNDJSONLogs(t *testing.T) { sp, err := conn.SpacePost(context.Background(), api.SpacePostRequest{Name: "test"}) require.NoError(t, err) - opts := &client.LogPostOpts{JSON: &tc} - _, err = conn.LogPostReaders(context.Background(), sp.ID, opts, strings.NewReader(src)) + _, err = conn.LogPostReaders(context.Background(), sp.ID, nil, strings.NewReader(src)) require.NoError(t, err) res := searchZson(t, conn, sp.ID, "*") require.Equal(t, expected, strings.TrimSpace(res)) - span := nano.Span{Ts: 1e9, Dur: 1e9 + 1} info, err := conn.SpaceInfo(context.Background(), sp.ID) require.NoError(t, err) require.Equal(t, &api.SpaceInfo{ @@ -515,8 +491,7 @@ func TestPostNDJSONLogs(t *testing.T) { DataPath: sp.DataPath, StorageKind: api.DefaultStorageKind(), }, - Span: &span, - Size: 79, + Size: 81, PcapSupport: false, }, info) } @@ -533,41 +508,6 @@ func TestPostNDJSONLogs(t *testing.T) { }) } -func TestPostNDJSONLogWarning(t *testing.T) { - src1 := strings.NewReader(`{"ts":"1000","_path":"http"} -{"ts":"2000","_path":"nosuchpath"}`) - src2 := strings.NewReader(`{"ts":"1000","_path":"http"} -{"ts":"1000","_path":"http","extra":"foo"}`) - tc := ndjsonio.TypeConfig{ - Descriptors: map[string][]interface{}{ - "http_log": []interface{}{ - map[string]interface{}{ - "name": "_path", - "type": "string", - }, - map[string]interface{}{ - "name": "ts", - "type": "time", - }, - }, - }, - Rules: []ndjsonio.Rule{ - ndjsonio.Rule{"_path", "http", "http_log"}, - }, - } - _, conn := newCore(t) - sp, err := conn.SpacePost(context.Background(), api.SpacePostRequest{Name: "test"}) - require.NoError(t, err) - - opts := &client.LogPostOpts{JSON: &tc} - res, err := conn.LogPostReaders(context.Background(), sp.ID, opts, src1, src2) - require.NoError(t, err) - require.Len(t, res.Warnings, 2) - assert.Regexp(t, ": line 2: descriptor not found", res.Warnings[0]) - assert.Regexp(t, ": line 2: incomplete descriptor", res.Warnings[1]) - assert.EqualValues(t, 134, res.BytesRead) -} - // Other attempts at malformed ZSON closer to the original are not yet flagged // as errors. See https://github.com/brimdata/zed/issues/2057#issuecomment-803148819 func TestPostLogStopErr(t *testing.T) { diff --git a/ppl/zqd/ingest/log.go b/ppl/zqd/ingest/log.go index f7bbafdcfc..cf32e800f6 100644 --- a/ppl/zqd/ingest/log.go +++ b/ppl/zqd/ingest/log.go @@ -15,7 +15,6 @@ import ( "github.com/brimdata/zed/zbuf" "github.com/brimdata/zed/zio" "github.com/brimdata/zed/zio/detector" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/zngio" "github.com/brimdata/zed/zqe" "github.com/brimdata/zed/zson" @@ -45,15 +44,6 @@ func NewLogOp(ctx context.Context, store storage.Storage, req api.LogPostRequest zctx: zson.NewContext(), } opts := zio.ReaderOpts{Zng: zngio.ReaderOpts{Validate: true}} - //XXX if there is a json config, then the input has to be ndjson - // and we shouldn't be using the detector. down the line, when - // we more holistically support json ingest, there should probably - // be a different endpoint or other params here which might select - // the input rules by a named configuration - if req.JSONTypeConfig != nil { - opts.JSON.TypeConfig = req.JSONTypeConfig - opts.JSON.PathRegexp = ndjsonio.DefaultPathRegexp - } proc, err := ast.UnpackJSONAsProc(req.Shaper) if err != nil { return nil, err diff --git a/ppl/zqd/ingest/multipartlogreader.go b/ppl/zqd/ingest/multipartlogreader.go index fda5c3f24b..624679a499 100644 --- a/ppl/zqd/ingest/multipartlogreader.go +++ b/ppl/zqd/ingest/multipartlogreader.go @@ -2,7 +2,6 @@ package ingest import ( "context" - "encoding/json" "fmt" "io" "io/ioutil" @@ -14,7 +13,6 @@ import ( "github.com/brimdata/zed/zbuf" "github.com/brimdata/zed/zio" "github.com/brimdata/zed/zio/detector" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/zngio" "github.com/brimdata/zed/zng" "github.com/brimdata/zed/zqe" @@ -83,13 +81,6 @@ next: } return nil, err } - if part.FormName() == "json_config" { - if err := json.NewDecoder(part).Decode(&m.opts.JSON.TypeConfig); err != nil { - return nil, zqe.ErrInvalid("bad typing config: %v", err) - } - m.opts.JSON.PathRegexp = ndjsonio.DefaultPathRegexp - goto next - } if part.FormName() == "shaper_ast" { raw, err := ioutil.ReadAll(io.LimitReader(part, maxShaperAstBytes)) if err != nil { diff --git a/zeek/README.md b/zeek/README.md index 00e0979cde..b4fd7f6f28 100644 --- a/zeek/README.md +++ b/zeek/README.md @@ -14,7 +14,7 @@ # Summary -The file [`types.json`](types.json) in this directory contains configuration +The file `types.json` in this directory contains configuration that can be used to restore Zeek-compatible rich data types to events that have been output in JSON format by Zeek. The definitions in the current revision of `types.json` reflect the default set of logs output by diff --git a/zeek/types.json b/zeek/types.json deleted file mode 100644 index e0fd5bc91b..0000000000 --- a/zeek/types.json +++ /dev/null @@ -1,3812 +0,0 @@ -{ - "descriptors": { - "broker_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "ty", - "type": "zenum" - }, - { - "name": "ev", - "type": "bstring" - }, - { - "name": "peer", - "type": [ - { - "name": "address", - "type": "bstring" - }, - { - "name": "bound_port", - "type": "port" - } - ] - }, - { - "name": "message", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "capture_loss_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "ts_delta", - "type": "duration" - }, - { - "name": "peer", - "type": "bstring" - }, - { - "name": "gaps", - "type": "uint64" - }, - { - "name": "acks", - "type": "uint64" - }, - { - "name": "percent_lost", - "type": "float64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "cluster_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "node", - "type": "bstring" - }, - { - "name": "message", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "config_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "id", - "type": "bstring" - }, - { - "name": "old_value", - "type": "bstring" - }, - { - "name": "new_value", - "type": "bstring" - }, - { - "name": "location", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "conn_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "service", - "type": "bstring" - }, - { - "name": "duration", - "type": "duration" - }, - { - "name": "orig_bytes", - "type": "uint64" - }, - { - "name": "resp_bytes", - "type": "uint64" - }, - { - "name": "conn_state", - "type": "bstring" - }, - { - "name": "local_orig", - "type": "bool" - }, - { - "name": "local_resp", - "type": "bool" - }, - { - "name": "missed_bytes", - "type": "uint64" - }, - { - "name": "history", - "type": "bstring" - }, - { - "name": "orig_pkts", - "type": "uint64" - }, - { - "name": "orig_ip_bytes", - "type": "uint64" - }, - { - "name": "resp_pkts", - "type": "uint64" - }, - { - "name": "resp_ip_bytes", - "type": "uint64" - }, - { - "name": "tunnel_parents", - "type": "set[bstring]" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "dce_rpc_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "rtt", - "type": "duration" - }, - { - "name": "named_pipe", - "type": "bstring" - }, - { - "name": "endpoint", - "type": "bstring" - }, - { - "name": "operation", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "dhcp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uids", - "type": "set[bstring]" - }, - { - "name": "client_addr", - "type": "ip" - }, - { - "name": "server_addr", - "type": "ip" - }, - { - "name": "mac", - "type": "bstring" - }, - { - "name": "host_name", - "type": "bstring" - }, - { - "name": "client_fqdn", - "type": "bstring" - }, - { - "name": "domain", - "type": "bstring" - }, - { - "name": "requested_addr", - "type": "ip" - }, - { - "name": "assigned_addr", - "type": "ip" - }, - { - "name": "lease_time", - "type": "duration" - }, - { - "name": "client_message", - "type": "bstring" - }, - { - "name": "server_message", - "type": "bstring" - }, - { - "name": "msg_types", - "type": "array[bstring]" - }, - { - "name": "duration", - "type": "duration" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "dnp3_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "fc_request", - "type": "bstring" - }, - { - "name": "fc_reply", - "type": "bstring" - }, - { - "name": "iin", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "dns_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "trans_id", - "type": "uint64" - }, - { - "name": "rtt", - "type": "duration" - }, - { - "name": "query", - "type": "bstring" - }, - { - "name": "qclass", - "type": "uint64" - }, - { - "name": "qclass_name", - "type": "bstring" - }, - { - "name": "qtype", - "type": "uint64" - }, - { - "name": "qtype_name", - "type": "bstring" - }, - { - "name": "rcode", - "type": "uint64" - }, - { - "name": "rcode_name", - "type": "bstring" - }, - { - "name": "AA", - "type": "bool" - }, - { - "name": "TC", - "type": "bool" - }, - { - "name": "RD", - "type": "bool" - }, - { - "name": "RA", - "type": "bool" - }, - { - "name": "Z", - "type": "uint64" - }, - { - "name": "answers", - "type": "array[bstring]" - }, - { - "name": "TTLs", - "type": "array[duration]" - }, - { - "name": "rejected", - "type": "bool" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "dpd_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "analyzer", - "type": "bstring" - }, - { - "name": "failure_reason", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "files_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "tx_hosts", - "type": "set[ip]" - }, - { - "name": "rx_hosts", - "type": "set[ip]" - }, - { - "name": "conn_uids", - "type": "set[bstring]" - }, - { - "name": "source", - "type": "bstring" - }, - { - "name": "depth", - "type": "uint64" - }, - { - "name": "analyzers", - "type": "set[bstring]" - }, - { - "name": "mime_type", - "type": "bstring" - }, - { - "name": "filename", - "type": "bstring" - }, - { - "name": "duration", - "type": "duration" - }, - { - "name": "local_orig", - "type": "bool" - }, - { - "name": "is_orig", - "type": "bool" - }, - { - "name": "seen_bytes", - "type": "uint64" - }, - { - "name": "total_bytes", - "type": "uint64" - }, - { - "name": "missing_bytes", - "type": "uint64" - }, - { - "name": "overflow_bytes", - "type": "uint64" - }, - { - "name": "timedout", - "type": "bool" - }, - { - "name": "parent_fuid", - "type": "bstring" - }, - { - "name": "md5", - "type": "bstring" - }, - { - "name": "sha1", - "type": "bstring" - }, - { - "name": "sha256", - "type": "bstring" - }, - { - "name": "extracted", - "type": "bstring" - }, - { - "name": "extracted_cutoff", - "type": "bool" - }, - { - "name": "extracted_size", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "ftp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "user", - "type": "bstring" - }, - { - "name": "password", - "type": "bstring" - }, - { - "name": "command", - "type": "bstring" - }, - { - "name": "arg", - "type": "bstring" - }, - { - "name": "mime_type", - "type": "bstring" - }, - { - "name": "file_size", - "type": "uint64" - }, - { - "name": "reply_code", - "type": "uint64" - }, - { - "name": "reply_msg", - "type": "bstring" - }, - { - "name": "data_channel", - "type": [ - { - "name": "passive", - "type": "bool" - }, - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "trans_depth", - "type": "uint64" - }, - { - "name": "method", - "type": "bstring" - }, - { - "name": "host", - "type": "bstring" - }, - { - "name": "uri", - "type": "bstring" - }, - { - "name": "referrer", - "type": "bstring" - }, - { - "name": "version", - "type": "bstring" - }, - { - "name": "user_agent", - "type": "bstring" - }, - { - "name": "origin", - "type": "bstring" - }, - { - "name": "request_body_len", - "type": "uint64" - }, - { - "name": "response_body_len", - "type": "uint64" - }, - { - "name": "status_code", - "type": "uint64" - }, - { - "name": "status_msg", - "type": "bstring" - }, - { - "name": "info_code", - "type": "uint64" - }, - { - "name": "info_msg", - "type": "bstring" - }, - { - "name": "tags", - "type": "set[zenum]" - }, - { - "name": "username", - "type": "bstring" - }, - { - "name": "password", - "type": "bstring" - }, - { - "name": "proxied", - "type": "set[bstring]" - }, - { - "name": "orig_fuids", - "type": "array[bstring]" - }, - { - "name": "orig_filenames", - "type": "array[bstring]" - }, - { - "name": "orig_mime_types", - "type": "array[bstring]" - }, - { - "name": "resp_fuids", - "type": "array[bstring]" - }, - { - "name": "resp_filenames", - "type": "array[bstring]" - }, - { - "name": "resp_mime_types", - "type": "array[bstring]" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "intel_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "seen", - "type": [ - { - "name": "indicator", - "type": "bstring" - }, - { - "name": "indicator_type", - "type": "zenum" - }, - { - "name": "where", - "type": "zenum" - }, - { - "name": "node", - "type": "bstring" - } - ] - }, - { - "name": "matched", - "type": "set[zenum]" - }, - { - "name": "sources", - "type": "set[bstring]" - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "file_mime_type", - "type": "bstring" - }, - { - "name": "file_desc", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "irc_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "nick", - "type": "bstring" - }, - { - "name": "user", - "type": "bstring" - }, - { - "name": "command", - "type": "bstring" - }, - { - "name": "value", - "type": "bstring" - }, - { - "name": "addl", - "type": "bstring" - }, - { - "name": "dcc_file_name", - "type": "bstring" - }, - { - "name": "dcc_file_size", - "type": "uint64" - }, - { - "name": "dcc_mime_type", - "type": "bstring" - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "kerberos_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "request_type", - "type": "bstring" - }, - { - "name": "client", - "type": "bstring" - }, - { - "name": "service", - "type": "bstring" - }, - { - "name": "success", - "type": "bool" - }, - { - "name": "error_msg", - "type": "bstring" - }, - { - "name": "from", - "type": "time" - }, - { - "name": "till", - "type": "time" - }, - { - "name": "cipher", - "type": "bstring" - }, - { - "name": "forwardable", - "type": "bool" - }, - { - "name": "renewable", - "type": "bool" - }, - { - "name": "client_cert_subject", - "type": "bstring" - }, - { - "name": "client_cert_fuid", - "type": "bstring" - }, - { - "name": "server_cert_subject", - "type": "bstring" - }, - { - "name": "server_cert_fuid", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "known_certs_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "host", - "type": "ip" - }, - { - "name": "port_num", - "type": "port" - }, - { - "name": "subject", - "type": "bstring" - }, - { - "name": "issuer_subject", - "type": "bstring" - }, - { - "name": "serial", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "known_hosts_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "host", - "type": "ip" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "known_services_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "host", - "type": "ip" - }, - { - "name": "port_num", - "type": "port" - }, - { - "name": "port_proto", - "type": "zenum" - }, - { - "name": "service", - "type": "set[bstring]" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "loaded_scripts_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "name", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "modbus_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "func", - "type": "bstring" - }, - { - "name": "exception", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "mysql_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "cmd", - "type": "bstring" - }, - { - "name": "arg", - "type": "bstring" - }, - { - "name": "success", - "type": "bool" - }, - { - "name": "rows", - "type": "uint64" - }, - { - "name": "response", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "netcontrol_drop_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "rule_id", - "type": "bstring" - }, - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - }, - { - "name": "expire", - "type": "duration" - }, - { - "name": "location", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "netcontrol_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "rule_id", - "type": "bstring" - }, - { - "name": "category", - "type": "zenum" - }, - { - "name": "cmd", - "type": "bstring" - }, - { - "name": "state", - "type": "zenum" - }, - { - "name": "action", - "type": "bstring" - }, - { - "name": "target", - "type": "zenum" - }, - { - "name": "entity_type", - "type": "bstring" - }, - { - "name": "entity", - "type": "bstring" - }, - { - "name": "mod", - "type": "bstring" - }, - { - "name": "msg", - "type": "bstring" - }, - { - "name": "priority", - "type": "int64" - }, - { - "name": "expire", - "type": "duration" - }, - { - "name": "location", - "type": "bstring" - }, - { - "name": "plugin", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "netcontrol_shunt_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "rule_id", - "type": "bstring" - }, - { - "name": "f", - "type": [ - { - "name": "src_h", - "type": "ip" - }, - { - "name": "src_p", - "type": "port" - }, - { - "name": "dst_h", - "type": "ip" - }, - { - "name": "dst_p", - "type": "port" - } - ] - }, - { - "name": "expire", - "type": "duration" - }, - { - "name": "location", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "notice_alarm_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "file_mime_type", - "type": "bstring" - }, - { - "name": "file_desc", - "type": "bstring" - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "note", - "type": "zenum" - }, - { - "name": "msg", - "type": "bstring" - }, - { - "name": "sub", - "type": "bstring" - }, - { - "name": "src", - "type": "ip" - }, - { - "name": "dst", - "type": "ip" - }, - { - "name": "p", - "type": "port" - }, - { - "name": "n", - "type": "uint64" - }, - { - "name": "peer_descr", - "type": "bstring" - }, - { - "name": "actions", - "type": "set[zenum]" - }, - { - "name": "suppress_for", - "type": "duration" - }, - { - "name": "remote_location", - "type": [ - { - "name": "country_code", - "type": "bstring" - }, - { - "name": "region", - "type": "bstring" - }, - { - "name": "city", - "type": "bstring" - }, - { - "name": "latitude", - "type": "float64" - }, - { - "name": "longitude", - "type": "float64" - } - ] - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "notice_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "file_mime_type", - "type": "bstring" - }, - { - "name": "file_desc", - "type": "bstring" - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "note", - "type": "zenum" - }, - { - "name": "msg", - "type": "bstring" - }, - { - "name": "sub", - "type": "bstring" - }, - { - "name": "src", - "type": "ip" - }, - { - "name": "dst", - "type": "ip" - }, - { - "name": "p", - "type": "port" - }, - { - "name": "n", - "type": "uint64" - }, - { - "name": "peer_descr", - "type": "bstring" - }, - { - "name": "actions", - "type": "set[zenum]" - }, - { - "name": "suppress_for", - "type": "duration" - }, - { - "name": "remote_location", - "type": [ - { - "name": "country_code", - "type": "bstring" - }, - { - "name": "region", - "type": "bstring" - }, - { - "name": "city", - "type": "bstring" - }, - { - "name": "latitude", - "type": "float64" - }, - { - "name": "longitude", - "type": "float64" - } - ] - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "ntlm_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "username", - "type": "bstring" - }, - { - "name": "hostname", - "type": "bstring" - }, - { - "name": "domainname", - "type": "bstring" - }, - { - "name": "server_nb_computer_name", - "type": "bstring" - }, - { - "name": "server_dns_computer_name", - "type": "bstring" - }, - { - "name": "server_tree_name", - "type": "bstring" - }, - { - "name": "success", - "type": "bool" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "ntp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "version", - "type": "uint64" - }, - { - "name": "mode", - "type": "uint64" - }, - { - "name": "stratum", - "type": "uint64" - }, - { - "name": "poll", - "type": "duration" - }, - { - "name": "precision", - "type": "duration" - }, - { - "name": "root_delay", - "type": "duration" - }, - { - "name": "root_disp", - "type": "duration" - }, - { - "name": "ref_id", - "type": "bstring" - }, - { - "name": "ref_time", - "type": "time" - }, - { - "name": "org_time", - "type": "time" - }, - { - "name": "rec_time", - "type": "time" - }, - { - "name": "xmt_time", - "type": "time" - }, - { - "name": "num_exts", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "packet_filter_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "node", - "type": "bstring" - }, - { - "name": "filter", - "type": "bstring" - }, - { - "name": "init", - "type": "bool" - }, - { - "name": "success", - "type": "bool" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "pe_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "id", - "type": "bstring" - }, - { - "name": "machine", - "type": "bstring" - }, - { - "name": "compile_ts", - "type": "time" - }, - { - "name": "os", - "type": "bstring" - }, - { - "name": "subsystem", - "type": "bstring" - }, - { - "name": "is_exe", - "type": "bool" - }, - { - "name": "is_64bit", - "type": "bool" - }, - { - "name": "uses_aslr", - "type": "bool" - }, - { - "name": "uses_dep", - "type": "bool" - }, - { - "name": "uses_code_integrity", - "type": "bool" - }, - { - "name": "uses_seh", - "type": "bool" - }, - { - "name": "has_import_table", - "type": "bool" - }, - { - "name": "has_export_table", - "type": "bool" - }, - { - "name": "has_cert_table", - "type": "bool" - }, - { - "name": "has_debug_data", - "type": "bool" - }, - { - "name": "section_names", - "type": "array[bstring]" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "radius_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "username", - "type": "bstring" - }, - { - "name": "mac", - "type": "bstring" - }, - { - "name": "framed_addr", - "type": "ip" - }, - { - "name": "tunnel_client", - "type": "bstring" - }, - { - "name": "connect_info", - "type": "bstring" - }, - { - "name": "reply_msg", - "type": "bstring" - }, - { - "name": "result", - "type": "bstring" - }, - { - "name": "ttl", - "type": "duration" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "rdp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "cookie", - "type": "bstring" - }, - { - "name": "result", - "type": "bstring" - }, - { - "name": "security_protocol", - "type": "bstring" - }, - { - "name": "client_channels", - "type": "array[bstring]" - }, - { - "name": "keyboard_layout", - "type": "bstring" - }, - { - "name": "client_build", - "type": "bstring" - }, - { - "name": "client_name", - "type": "bstring" - }, - { - "name": "client_dig_product_id", - "type": "bstring" - }, - { - "name": "desktop_width", - "type": "uint64" - }, - { - "name": "desktop_height", - "type": "uint64" - }, - { - "name": "requested_color_depth", - "type": "bstring" - }, - { - "name": "cert_type", - "type": "bstring" - }, - { - "name": "cert_count", - "type": "uint64" - }, - { - "name": "cert_permanent", - "type": "bool" - }, - { - "name": "encryption_level", - "type": "bstring" - }, - { - "name": "encryption_method", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "reporter_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "level", - "type": "zenum" - }, - { - "name": "message", - "type": "bstring" - }, - { - "name": "location", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "rfb_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "client_major_version", - "type": "bstring" - }, - { - "name": "client_minor_version", - "type": "bstring" - }, - { - "name": "server_major_version", - "type": "bstring" - }, - { - "name": "server_minor_version", - "type": "bstring" - }, - { - "name": "authentication_method", - "type": "bstring" - }, - { - "name": "auth", - "type": "bool" - }, - { - "name": "share_flag", - "type": "bool" - }, - { - "name": "desktop_name", - "type": "bstring" - }, - { - "name": "width", - "type": "uint64" - }, - { - "name": "height", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "signatures_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "src_addr", - "type": "ip" - }, - { - "name": "src_port", - "type": "port" - }, - { - "name": "dst_addr", - "type": "ip" - }, - { - "name": "dst_port", - "type": "port" - }, - { - "name": "note", - "type": "zenum" - }, - { - "name": "sig_id", - "type": "bstring" - }, - { - "name": "event_msg", - "type": "bstring" - }, - { - "name": "sub_msg", - "type": "bstring" - }, - { - "name": "sig_count", - "type": "uint64" - }, - { - "name": "host_count", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "sip_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "trans_depth", - "type": "uint64" - }, - { - "name": "method", - "type": "bstring" - }, - { - "name": "uri", - "type": "bstring" - }, - { - "name": "date", - "type": "bstring" - }, - { - "name": "request_from", - "type": "bstring" - }, - { - "name": "request_to", - "type": "bstring" - }, - { - "name": "response_from", - "type": "bstring" - }, - { - "name": "response_to", - "type": "bstring" - }, - { - "name": "reply_to", - "type": "bstring" - }, - { - "name": "call_id", - "type": "bstring" - }, - { - "name": "seq", - "type": "bstring" - }, - { - "name": "subject", - "type": "bstring" - }, - { - "name": "request_path", - "type": "array[bstring]" - }, - { - "name": "response_path", - "type": "array[bstring]" - }, - { - "name": "user_agent", - "type": "bstring" - }, - { - "name": "status_code", - "type": "uint64" - }, - { - "name": "status_msg", - "type": "bstring" - }, - { - "name": "warning", - "type": "bstring" - }, - { - "name": "request_body_len", - "type": "uint64" - }, - { - "name": "response_body_len", - "type": "uint64" - }, - { - "name": "content_type", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "smb_files_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "fuid", - "type": "bstring" - }, - { - "name": "action", - "type": "zenum" - }, - { - "name": "path", - "type": "bstring" - }, - { - "name": "name", - "type": "bstring" - }, - { - "name": "size", - "type": "uint64" - }, - { - "name": "prev_name", - "type": "bstring" - }, - { - "name": "times", - "type": [ - { - "name": "modified", - "type": "time" - }, - { - "name": "accessed", - "type": "time" - }, - { - "name": "created", - "type": "time" - }, - { - "name": "changed", - "type": "time" - } - ] - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "smb_mapping_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "path", - "type": "bstring" - }, - { - "name": "service", - "type": "bstring" - }, - { - "name": "native_file_system", - "type": "bstring" - }, - { - "name": "share_type", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "smtp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "trans_depth", - "type": "uint64" - }, - { - "name": "helo", - "type": "bstring" - }, - { - "name": "mailfrom", - "type": "bstring" - }, - { - "name": "rcptto", - "type": "set[bstring]" - }, - { - "name": "date", - "type": "bstring" - }, - { - "name": "from", - "type": "bstring" - }, - { - "name": "to", - "type": "set[bstring]" - }, - { - "name": "cc", - "type": "set[bstring]" - }, - { - "name": "reply_to", - "type": "bstring" - }, - { - "name": "msg_id", - "type": "bstring" - }, - { - "name": "in_reply_to", - "type": "bstring" - }, - { - "name": "subject", - "type": "bstring" - }, - { - "name": "x_originating_ip", - "type": "ip" - }, - { - "name": "first_received", - "type": "bstring" - }, - { - "name": "second_received", - "type": "bstring" - }, - { - "name": "last_reply", - "type": "bstring" - }, - { - "name": "path", - "type": "array[ip]" - }, - { - "name": "user_agent", - "type": "bstring" - }, - { - "name": "tls", - "type": "bool" - }, - { - "name": "fuids", - "type": "array[bstring]" - }, - { - "name": "is_webmail", - "type": "bool" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "snmp_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "duration", - "type": "duration" - }, - { - "name": "version", - "type": "bstring" - }, - { - "name": "community", - "type": "bstring" - }, - { - "name": "get_requests", - "type": "uint64" - }, - { - "name": "get_bulk_requests", - "type": "uint64" - }, - { - "name": "get_responses", - "type": "uint64" - }, - { - "name": "set_requests", - "type": "uint64" - }, - { - "name": "display_string", - "type": "bstring" - }, - { - "name": "up_since", - "type": "time" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "socks_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "version", - "type": "uint64" - }, - { - "name": "user", - "type": "bstring" - }, - { - "name": "password", - "type": "bstring" - }, - { - "name": "status", - "type": "bstring" - }, - { - "name": "request", - "type": [ - { - "name": "host", - "type": "ip" - }, - { - "name": "name", - "type": "bstring" - } - ] - }, - { - "name": "request_p", - "type": "port" - }, - { - "name": "bound", - "type": [ - { - "name": "host", - "type": "ip" - }, - { - "name": "name", - "type": "bstring" - } - ] - }, - { - "name": "bound_p", - "type": "port" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "software_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "host", - "type": "ip" - }, - { - "name": "host_p", - "type": "port" - }, - { - "name": "software_type", - "type": "zenum" - }, - { - "name": "name", - "type": "bstring" - }, - { - "name": "version", - "type": [ - { - "name": "major", - "type": "uint64" - }, - { - "name": "minor", - "type": "uint64" - }, - { - "name": "minor2", - "type": "uint64" - }, - { - "name": "minor3", - "type": "uint64" - }, - { - "name": "addl", - "type": "bstring" - } - ] - }, - { - "name": "unparsed_version", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "ssh_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "version", - "type": "uint64" - }, - { - "name": "auth_success", - "type": "bool" - }, - { - "name": "auth_attempts", - "type": "uint64" - }, - { - "name": "direction", - "type": "zenum" - }, - { - "name": "client", - "type": "bstring" - }, - { - "name": "server", - "type": "bstring" - }, - { - "name": "cipher_alg", - "type": "bstring" - }, - { - "name": "mac_alg", - "type": "bstring" - }, - { - "name": "compression_alg", - "type": "bstring" - }, - { - "name": "kex_alg", - "type": "bstring" - }, - { - "name": "host_key_alg", - "type": "bstring" - }, - { - "name": "host_key", - "type": "bstring" - }, - { - "name": "remote_location", - "type": [ - { - "name": "country_code", - "type": "bstring" - }, - { - "name": "region", - "type": "bstring" - }, - { - "name": "city", - "type": "bstring" - }, - { - "name": "latitude", - "type": "float64" - }, - { - "name": "longitude", - "type": "float64" - } - ] - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "ssl_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "version", - "type": "bstring" - }, - { - "name": "cipher", - "type": "bstring" - }, - { - "name": "curve", - "type": "bstring" - }, - { - "name": "server_name", - "type": "bstring" - }, - { - "name": "resumed", - "type": "bool" - }, - { - "name": "last_alert", - "type": "bstring" - }, - { - "name": "next_protocol", - "type": "bstring" - }, - { - "name": "established", - "type": "bool" - }, - { - "name": "cert_chain_fuids", - "type": "array[bstring]" - }, - { - "name": "client_cert_chain_fuids", - "type": "array[bstring]" - }, - { - "name": "subject", - "type": "bstring" - }, - { - "name": "issuer", - "type": "bstring" - }, - { - "name": "client_subject", - "type": "bstring" - }, - { - "name": "client_issuer", - "type": "bstring" - }, - { - "name": "validation_status", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "stats_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "peer", - "type": "bstring" - }, - { - "name": "mem", - "type": "uint64" - }, - { - "name": "pkts_proc", - "type": "uint64" - }, - { - "name": "bytes_recv", - "type": "uint64" - }, - { - "name": "pkts_dropped", - "type": "uint64" - }, - { - "name": "pkts_link", - "type": "uint64" - }, - { - "name": "pkt_lag", - "type": "duration" - }, - { - "name": "events_proc", - "type": "uint64" - }, - { - "name": "events_queued", - "type": "uint64" - }, - { - "name": "active_tcp_conns", - "type": "uint64" - }, - { - "name": "active_udp_conns", - "type": "uint64" - }, - { - "name": "active_icmp_conns", - "type": "uint64" - }, - { - "name": "tcp_conns", - "type": "uint64" - }, - { - "name": "udp_conns", - "type": "uint64" - }, - { - "name": "icmp_conns", - "type": "uint64" - }, - { - "name": "timers", - "type": "uint64" - }, - { - "name": "active_timers", - "type": "uint64" - }, - { - "name": "files", - "type": "uint64" - }, - { - "name": "active_files", - "type": "uint64" - }, - { - "name": "dns_requests", - "type": "uint64" - }, - { - "name": "active_dns_requests", - "type": "uint64" - }, - { - "name": "reassem_tcp_size", - "type": "uint64" - }, - { - "name": "reassem_file_size", - "type": "uint64" - }, - { - "name": "reassem_frag_size", - "type": "uint64" - }, - { - "name": "reassem_unknown_size", - "type": "uint64" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "syslog_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "proto", - "type": "zenum" - }, - { - "name": "facility", - "type": "bstring" - }, - { - "name": "severity", - "type": "bstring" - }, - { - "name": "message", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "tunnel_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "tunnel_type", - "type": "zenum" - }, - { - "name": "action", - "type": "zenum" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "weird_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - }, - { - "name": "orig_p", - "type": "port" - }, - { - "name": "resp_h", - "type": "ip" - }, - { - "name": "resp_p", - "type": "port" - } - ] - }, - { - "name": "name", - "type": "bstring" - }, - { - "name": "addl", - "type": "bstring" - }, - { - "name": "notice", - "type": "bool" - }, - { - "name": "peer", - "type": "bstring" - }, - { - "name": "source", - "type": "bstring" - }, - { - "name": "_write_ts", - "type": "time" - } - ], - "x509_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "id", - "type": "bstring" - }, - { - "name": "certificate", - "type": [ - { - "name": "version", - "type": "uint64" - }, - { - "name": "serial", - "type": "bstring" - }, - { - "name": "subject", - "type": "bstring" - }, - { - "name": "issuer", - "type": "bstring" - }, - { - "name": "not_valid_before", - "type": "time" - }, - { - "name": "not_valid_after", - "type": "time" - }, - { - "name": "key_alg", - "type": "bstring" - }, - { - "name": "sig_alg", - "type": "bstring" - }, - { - "name": "key_type", - "type": "bstring" - }, - { - "name": "key_length", - "type": "uint64" - }, - { - "name": "exponent", - "type": "bstring" - }, - { - "name": "curve", - "type": "bstring" - } - ] - }, - { - "name": "san", - "type": [ - { - "name": "dns", - "type": "array[bstring]" - }, - { - "name": "uri", - "type": "array[bstring]" - }, - { - "name": "email", - "type": "array[bstring]" - }, - { - "name": "ip", - "type": "array[ip]" - } - ] - }, - { - "name": "basic_constraints", - "type": [ - { - "name": "ca", - "type": "bool" - }, - { - "name": "path_len", - "type": "uint64" - } - ] - }, - { - "name": "_write_ts", - "type": "time" - } - ] - }, - "rules": [ - { - "descriptor": "broker_log", - "name": "_path", - "value": "broker" - }, - { - "descriptor": "capture_loss_log", - "name": "_path", - "value": "capture_loss" - }, - { - "descriptor": "cluster_log", - "name": "_path", - "value": "cluster" - }, - { - "descriptor": "config_log", - "name": "_path", - "value": "config" - }, - { - "descriptor": "conn_log", - "name": "_path", - "value": "conn" - }, - { - "descriptor": "dce_rpc_log", - "name": "_path", - "value": "dce_rpc" - }, - { - "descriptor": "dhcp_log", - "name": "_path", - "value": "dhcp" - }, - { - "descriptor": "dnp3_log", - "name": "_path", - "value": "dnp3" - }, - { - "descriptor": "dns_log", - "name": "_path", - "value": "dns" - }, - { - "descriptor": "dpd_log", - "name": "_path", - "value": "dpd" - }, - { - "descriptor": "files_log", - "name": "_path", - "value": "files" - }, - { - "descriptor": "ftp_log", - "name": "_path", - "value": "ftp" - }, - { - "descriptor": "http_log", - "name": "_path", - "value": "http" - }, - { - "descriptor": "intel_log", - "name": "_path", - "value": "intel" - }, - { - "descriptor": "irc_log", - "name": "_path", - "value": "irc" - }, - { - "descriptor": "kerberos_log", - "name": "_path", - "value": "kerberos" - }, - { - "descriptor": "known_certs_log", - "name": "_path", - "value": "known_certs" - }, - { - "descriptor": "known_hosts_log", - "name": "_path", - "value": "known_hosts" - }, - { - "descriptor": "known_services_log", - "name": "_path", - "value": "known_services" - }, - { - "descriptor": "loaded_scripts_log", - "name": "_path", - "value": "loaded_scripts" - }, - { - "descriptor": "modbus_log", - "name": "_path", - "value": "modbus" - }, - { - "descriptor": "mysql_log", - "name": "_path", - "value": "mysql" - }, - { - "descriptor": "netcontrol_log", - "name": "_path", - "value": "netcontrol" - }, - { - "descriptor": "netcontrol_drop_log", - "name": "_path", - "value": "netcontrol_drop" - }, - { - "descriptor": "netcontrol_shunt_log", - "name": "_path", - "value": "netcontrol_shunt" - }, - { - "descriptor": "notice_log", - "name": "_path", - "value": "notice" - }, - { - "descriptor": "notice_alarm_log", - "name": "_path", - "value": "notice_alarm" - }, - { - "descriptor": "ntlm_log", - "name": "_path", - "value": "ntlm" - }, - { - "descriptor": "ntp_log", - "name": "_path", - "value": "ntp" - }, - { - "descriptor": "packet_filter_log", - "name": "_path", - "value": "packet_filter" - }, - { - "descriptor": "pe_log", - "name": "_path", - "value": "pe" - }, - { - "descriptor": "radius_log", - "name": "_path", - "value": "radius" - }, - { - "descriptor": "rdp_log", - "name": "_path", - "value": "rdp" - }, - { - "descriptor": "reporter_log", - "name": "_path", - "value": "reporter" - }, - { - "descriptor": "rfb_log", - "name": "_path", - "value": "rfb" - }, - { - "descriptor": "signatures_log", - "name": "_path", - "value": "signatures" - }, - { - "descriptor": "sip_log", - "name": "_path", - "value": "sip" - }, - { - "descriptor": "smb_files_log", - "name": "_path", - "value": "smb_files" - }, - { - "descriptor": "smb_mapping_log", - "name": "_path", - "value": "smb_mapping" - }, - { - "descriptor": "smtp_log", - "name": "_path", - "value": "smtp" - }, - { - "descriptor": "snmp_log", - "name": "_path", - "value": "snmp" - }, - { - "descriptor": "socks_log", - "name": "_path", - "value": "socks" - }, - { - "descriptor": "software_log", - "name": "_path", - "value": "software" - }, - { - "descriptor": "ssh_log", - "name": "_path", - "value": "ssh" - }, - { - "descriptor": "ssl_log", - "name": "_path", - "value": "ssl" - }, - { - "descriptor": "stats_log", - "name": "_path", - "value": "stats" - }, - { - "descriptor": "syslog_log", - "name": "_path", - "value": "syslog" - }, - { - "descriptor": "tunnel_log", - "name": "_path", - "value": "tunnel" - }, - { - "descriptor": "weird_log", - "name": "_path", - "value": "weird" - }, - { - "descriptor": "x509_log", - "name": "_path", - "value": "x509" - } - ] -} diff --git a/zio/detector/file.go b/zio/detector/file.go index ff2c8cbf5e..2f743bb58a 100644 --- a/zio/detector/file.go +++ b/zio/detector/file.go @@ -41,9 +41,9 @@ func OpenFromNamedReadCloser(zctx *zson.Context, rc io.ReadCloser, path string, } var zr zbuf.Reader if opts.Format == "" || opts.Format == "auto" { - zr, err = NewReaderWithOpts(r, zctx, path, opts) + zr, err = NewReaderWithOpts(r, zctx, opts) } else { - zr, err = lookupReader(r, zctx, path, opts) + zr, err = lookupReader(r, zctx, opts) } if err != nil { return nil, err diff --git a/zio/detector/lookup.go b/zio/detector/lookup.go index 2dc27db60c..ba71f4021d 100644 --- a/zio/detector/lookup.go +++ b/zio/detector/lookup.go @@ -71,7 +71,7 @@ func LookupWriter(w io.WriteCloser, opts zio.WriterOpts) (zbuf.WriteCloser, erro } } -func lookupReader(r io.Reader, zctx *zson.Context, path string, opts zio.ReaderOpts) (zbuf.Reader, error) { +func lookupReader(r io.Reader, zctx *zson.Context, opts zio.ReaderOpts) (zbuf.Reader, error) { switch opts.Format { case "csv": return csvio.NewReader(r, zctx), nil @@ -79,15 +79,13 @@ func lookupReader(r io.Reader, zctx *zson.Context, path string, opts zio.ReaderO return tzngio.NewReader(r, zctx), nil case "zeek": return zeekio.NewReader(r, zctx) - case "ndjson": - return ndjsonio.NewReader(r, zctx, opts.JSON, path) case "json": return jsonio.NewReader(r, zctx) case "zjson": return zjsonio.NewReader(r, zctx), nil case "zng": return zngio.NewReaderWithOpts(r, zctx, opts.Zng), nil - case "zson": + case "ndjson", "zson": return zson.NewReader(r, zctx), nil case "zst": return zstio.NewReader(r, zctx) diff --git a/zio/detector/reader.go b/zio/detector/reader.go index 6aac086de0..1335715daf 100644 --- a/zio/detector/reader.go +++ b/zio/detector/reader.go @@ -7,7 +7,6 @@ import ( "github.com/brimdata/zed/zbuf" "github.com/brimdata/zed/zio" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/tzngio" "github.com/brimdata/zed/zio/zeekio" "github.com/brimdata/zed/zio/zjsonio" @@ -16,7 +15,7 @@ import ( "github.com/brimdata/zed/zson" ) -func NewReaderWithOpts(r io.Reader, zctx *zson.Context, path string, opts zio.ReaderOpts) (zbuf.Reader, error) { +func NewReaderWithOpts(r io.Reader, zctx *zson.Context, opts zio.ReaderOpts) (zbuf.Reader, error) { recorder := NewRecorder(r) track := NewTrack(recorder) @@ -36,39 +35,13 @@ func NewReaderWithOpts(r io.Reader, zctx *zson.Context, path string, opts zio.Re } track.Reset() - // zjson must come before ndjson since zjson is a subset of ndjson + // ZJSON must come before ZSON since ZJSON is a subset of ZSON zjsonErr := match(zjsonio.NewReader(track, zson.NewContext()), "zjson") if zjsonErr == nil { return zjsonio.NewReader(recorder, zctx), nil } track.Reset() - // Only try NDJSON if there is an explicit config to control the NDJSON - // parser. Otherwise, if this is NDJSON, we will fall through below - // and match ZSON, which can decode NDJSON. If someone wants "strict" - // JSON parsing (i.e., treat all numbers as float64 not a mix of int64 - // and float64, then we should use -i zson with the forthcoming json - // strict config). - ndjsonErr := errors.New("no json type config: ndjson detector skipped") - if opts.JSON.TypeConfig != nil { - // ndjson must come after zjson since zjson is a subset of ndjson - nr, err := ndjsonio.NewReader(track, zson.NewContext(), opts.JSON, path) - if err != nil { - return nil, err - } - if err := match(nr, "ndjson"); err != nil { - // Some clients might attach a typeconfig even - // if they're not sending JSON. So try again - // without a typeconfig to do regular - // autodetection on all formats. - copyOpts := opts - copyOpts.JSON.TypeConfig = nil - return NewReaderWithOpts(recorder, zctx, path, copyOpts) - } - return ndjsonio.NewReader(recorder, zctx, opts.JSON, path) - } - - // ZSON comes after NDJSON since ZSON is a superset of JSON. zsonErr := match(zson.NewReader(track, zson.NewContext()), "zson") if zsonErr == nil { return zson.NewReader(recorder, zctx), nil @@ -93,11 +66,11 @@ func NewReaderWithOpts(r io.Reader, zctx *zson.Context, path string, opts zio.Re parquetErr := errors.New("parquet: auto-detection not supported") zstErr := errors.New("zst: auto-detection not supported") - return nil, joinErrs([]error{tzngErr, zeekErr, ndjsonErr, zjsonErr, zsonErr, zngErr, parquetErr, zstErr}) + return nil, joinErrs([]error{tzngErr, zeekErr, zjsonErr, zsonErr, zngErr, parquetErr, zstErr}) } func NewReader(r io.Reader, zctx *zson.Context) (zbuf.Reader, error) { - return NewReaderWithOpts(r, zctx, "", zio.ReaderOpts{}) + return NewReaderWithOpts(r, zctx, zio.ReaderOpts{}) } func joinErrs(errs []error) error { diff --git a/zio/jsonio/reader.go b/zio/jsonio/reader.go index 551a57cae0..5b0dc307ef 100644 --- a/zio/jsonio/reader.go +++ b/zio/jsonio/reader.go @@ -8,7 +8,6 @@ import ( "io" "io/ioutil" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zng" "github.com/brimdata/zed/zson" ) @@ -18,7 +17,6 @@ const MaxReadBuffer = 25 * 1024 * 1024 type Reader struct { zctx *zson.Context reader io.Reader - parser *ndjsonio.InferParser objects []interface{} } @@ -26,7 +24,6 @@ func NewReader(r io.Reader, zctx *zson.Context) (*Reader, error) { return &Reader{ zctx: zctx, reader: r, - parser: ndjsonio.NewInferParser(zctx), }, nil } diff --git a/zio/ndjsonio/compat/decoder.go b/zio/ndjsonio/compat/decoder.go deleted file mode 100644 index 5d547089c1..0000000000 --- a/zio/ndjsonio/compat/decoder.go +++ /dev/null @@ -1,47 +0,0 @@ -// Package compat is here for compatibility with the types.json format. -// Ndjsonio was using zjsonio's decoder but we changed the format so this -// dependency would otherwise break existing types.json files. When the -// type mapping subsystem is updated and improved, this compat layer can go away. -package compat - -import "errors" - -// decode a nested JSON object into a zeek type string and return the string. -func DecodeType(columns []interface{}) (string, error) { - s := "record[" - comma := "" - for _, o := range columns { - // each column a json object with name and type - m, ok := o.(map[string]interface{}) - if !ok { - return "", errors.New("zjson type not a json object") - } - nameObj, ok := m["name"] - if !ok { - return "", errors.New("zjson type object missing name field") - } - name, ok := nameObj.(string) - if !ok { - return "", errors.New("zjson type object has non-string name field") - } - typeObj, ok := m["type"] - if !ok { - return "", errors.New("zjson type object missing type field") - } - typeName, ok := typeObj.(string) - if !ok { - childColumns, ok := typeObj.([]interface{}) - if !ok { - return "", errors.New("zjson type field contains invalid type") - } - var err error - typeName, err = DecodeType(childColumns) - if err != nil { - return "", err - } - } - s += comma + name + ":" + typeName - comma = "," - } - return s + "]", nil -} diff --git a/zio/ndjsonio/config.go b/zio/ndjsonio/config.go deleted file mode 100644 index a0e6e37d1a..0000000000 --- a/zio/ndjsonio/config.go +++ /dev/null @@ -1,62 +0,0 @@ -package ndjsonio - -import ( - "fmt" -) - -// A Rule contains one or more matches and the name of a descriptor -// key (in the companion Descriptors map). -type Rule struct { - Name string `json:"name"` - Value string `json:"value"` - Descriptor string `json:"descriptor"` -} - -// A TypeConfig contains a map of Descriptors, keyed by name, and a -// list of rules defining which records should be mapped into which -// descriptor. -type TypeConfig struct { - // If PassUnknowns is true, additional fields not found in the - // descriptor are included after running regular 5-type json - // inference on them. Otherwise, additional fields result in - // an error. - PassUnknowns bool `json:"pass_unknowns"` - Descriptors map[string][]interface{} `json:"descriptors"` - Rules []Rule `json:"rules"` -} - -func hasField(name string, columns []interface{}) bool { - for _, colmap := range columns { - col := colmap.(map[string]interface{}) - if col["name"] == name { - return true - } - } - return false -} - -// Validate validates a typing config. -func (conf TypeConfig) Validate() error { - for _, rule := range conf.Rules { - d, ok := conf.Descriptors[rule.Descriptor] - if !ok { - return fmt.Errorf("rule %s=%s uses descriptor %s that does not exist", rule.Name, rule.Value, rule.Descriptor) - } - if !hasField(rule.Name, d) { - return fmt.Errorf("rule %s refers to field %s that is not present in descriptor", rule.Descriptor, rule.Name) - } - - } - for name, desc := range conf.Descriptors { - for i, d := range desc { - col, ok := d.(map[string]interface{}) - if !ok { - return fmt.Errorf("descriptor %s has invalid structure in element %d", name, i) - } - if col["name"] == "ts" && col["type"] != "time" { - return fmt.Errorf("descriptor %s has field ts with wrong type %s", name, col["type"]) - } - } - } - return nil -} diff --git a/zio/ndjsonio/config_test.go b/zio/ndjsonio/config_test.go deleted file mode 100644 index 3d6b3ba74b..0000000000 --- a/zio/ndjsonio/config_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package ndjsonio - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestTypeConfigValidate(t *testing.T) { - testcases := []struct { - name string - in string - ok bool - }{ - { - name: "Valid config", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "type": "time", - "name": "ts" - }, - { - "name": "id", - "type": { - "type": "ip", - "name": "orig_h" - } - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "capture_loss", - "descriptor": "capture_loss_log" - } - ] - } - `, - ok: true, - }, - { - name: "Invalid descriptor in matching rule", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "type": "time", - "name": "ts" - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "capture_loss", - "descriptor": "capture_loss" - } - ] - } - `, - ok: false, - }, - { - name: "Use of non-time ts field", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "type": "string", - "name": "ts" - } - ] - }, - "rules": [ - ] - } - `, - ok: false, - }, - { - name: "Matching rule refers to absent field", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "type": "time", - "name": "ts" - } - ] - }, - "rules": [ - { - "name": "path", - "value": "capture_loss", - "descriptor": "capture_loss_log" - } - ] - } - `, - ok: false, - }, - { - name: "Matching rule refers to absent field", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "type": "time", - "name": "ts" - } - ] - }, - "rules": [ - { - "name": "path", - "value": "capture_loss", - "descriptor": "capture_loss_log" - } - ] - } - `, - ok: false, - }, - { - name: "Descriptor without _path in first column", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "time", - "name": "ts" - } - ] - }, - "rules": [ - { - "name": "path", - "value": "capture_loss", - "descriptor": "capture_loss_log" - } - ] - } - `, - ok: false, - }, - { - name: "Descriptor with invalid structure", - in: ` - { - "descriptors": { - "capture_loss_log": [ - { - "type": "string", - "name": "_path" - }, - { - "d": ["time","ts"] - } - ] - }, - "rules": [ - { - "name": "path", - "value": "capture_loss", - "descriptor": "capture_loss_log" - } - ] - } - `, - ok: false, - }, - } - - for _, c := range testcases { - t.Run(c.name, func(t *testing.T) { - tc := TypeConfig{} - err := json.Unmarshal([]byte(c.in), &tc) - require.NoError(t, err) - err = tc.Validate() - if c.ok { - require.NoError(t, err) - } else { - require.Error(t, err) - } - }) - } -} diff --git a/zio/ndjsonio/inferparser.go b/zio/ndjsonio/inferparser.go deleted file mode 100644 index ba52dc61c1..0000000000 --- a/zio/ndjsonio/inferparser.go +++ /dev/null @@ -1,219 +0,0 @@ -package ndjsonio - -import ( - "bytes" - "fmt" - "sort" - - "github.com/brimdata/zed/field" - "github.com/brimdata/zed/pkg/byteconv" - "github.com/brimdata/zed/zcode" - "github.com/brimdata/zed/zio/tzngio" - "github.com/brimdata/zed/zng" - "github.com/brimdata/zed/zng/builder" - "github.com/brimdata/zed/zson" - "github.com/buger/jsonparser" -) - -type InferParser inferParser - -func NewInferParser(zctx *zson.Context) *InferParser { - return &InferParser{zctx} -} - -func (p *InferParser) ParseObject(b []byte) (zng.Value, error) { - return (*inferParser)(p).parseObject(b) -} - -type inferParser struct { - zctx *zson.Context -} - -func (p *inferParser) parseObject(b []byte) (zng.Value, error) { - type kv struct { - key []byte - value []byte - typ jsonparser.ValueType - } - var kvs []kv - err := jsonparser.ObjectEach(b, func(key []byte, value []byte, typ jsonparser.ValueType, offset int) error { - kvs = append(kvs, kv{key, value, typ}) - return nil - }) - if err != nil { - return zng.Value{}, err - } - if len(kvs) == 0 { - empty, err := p.zctx.LookupTypeRecord([]zng.Column{}) - if err != nil { - return zng.Value{}, err - } - return zng.Value{Type: empty, Bytes: zcode.Bytes{}}, nil - } - - // Sort fields lexigraphically ensuring maps with the same - // columns but different printed order get assigned the same descriptor. - sort.Slice(kvs, func(i, j int) bool { - return bytes.Compare(kvs[i].key, kvs[j].key) < 0 - }) - var fields []field.Static - var zngTypes []zng.Type - var zngValues []zng.Value - for _, kv := range kvs { - fields = append(fields, field.Dotted(string(kv.key))) - v, err := p.parseValue(kv.value, kv.typ) - if err != nil { - return zng.Value{}, err - } - zngTypes = append(zngTypes, v.Type) - zngValues = append(zngValues, v) - } - columnBuilder, err := builder.NewColumnBuilder(p.zctx, fields) - if err != nil { - return zng.Value{}, err - } - typ, err := p.zctx.LookupTypeRecord(columnBuilder.TypedColumns(zngTypes)) - if err != nil { - return zng.Value{}, err - } - for _, v := range zngValues { - columnBuilder.Append(v.Bytes, zng.IsContainerType(zng.AliasOf(v.Type))) - } - zbytes, err := columnBuilder.Encode() - if err != nil { - return zng.Value{}, err - } - return zng.Value{Type: typ, Bytes: zbytes}, nil -} - -func (p *inferParser) parseValue(raw []byte, typ jsonparser.ValueType) (zng.Value, error) { - switch typ { - case jsonparser.Array: - return p.parseArray(raw) - case jsonparser.Object: - return p.parseObject(raw) - case jsonparser.Boolean: - return p.parseBool(raw) - case jsonparser.Number: - return p.parseNumber(raw) - case jsonparser.Null: - return p.parseNull() - case jsonparser.String: - return p.parseString(raw) - default: - return zng.Value{}, fmt.Errorf("unsupported type %v", typ) - } -} - -func typeIndex(typs []zng.Type, typ zng.Type) int { - for i := range typs { - if typ == typs[i] { - return i - } - } - return -1 -} - -func (p *inferParser) unionType(vals []zng.Value) *zng.TypeUnion { - var typs []zng.Type - for i := range vals { - if index := typeIndex(typs, vals[i].Type); index == -1 { - typs = append(typs, vals[i].Type) - } - } - if len(typs) <= 1 { - return nil - } - return p.zctx.LookupTypeUnion(typs) -} - -func encodeUnionArray(typ *zng.TypeUnion, vals []zng.Value) zcode.Bytes { - var b zcode.Builder - for i := range vals { - b.BeginContainer() - index := typeIndex(typ.Types, vals[i].Type) - b.AppendPrimitive(zng.EncodeInt(int64(index))) - if zng.IsContainerType(vals[i].Type) { - b.AppendContainer(vals[i].Bytes) - } else { - b.AppendPrimitive(vals[i].Bytes) - } - b.EndContainer() - } - return b.Bytes() -} - -func encodeContainer(vals []zng.Value) zcode.Bytes { - b := zcode.Bytes{} - for i := range vals { - if zng.IsContainerType(vals[i].Type) { - b = zcode.AppendContainer(b, vals[i].Bytes) - } else { - b = zcode.AppendPrimitive(b, vals[i].Bytes) - } - } - return b -} - -func (p *inferParser) parseArray(raw []byte) (zng.Value, error) { - var err error - var vals []zng.Value - jsonparser.ArrayEach(raw, func(el []byte, typ jsonparser.ValueType, offset int, elErr error) { - if elErr != nil { - err = elErr - return - } - val, err := p.parseValue(el, typ) - if err != nil { - return - } - vals = append(vals, val) - }) - if err != nil { - return zng.Value{}, err - } - union := p.unionType(vals) - if union != nil { - typ := p.zctx.LookupTypeArray(union) - return zng.Value{typ, encodeUnionArray(union, vals)}, nil - } - var typ zng.Type - if len(vals) == 0 { - typ = p.zctx.LookupTypeArray(zng.TypeString) - } else { - typ = p.zctx.LookupTypeArray(vals[0].Type) - } - return zng.Value{typ, encodeContainer(vals)}, nil -} - -func (p *inferParser) parseBool(b []byte) (zng.Value, error) { - boolean, err := jsonparser.GetBoolean(b) - if err != nil { - return zng.Value{}, err - } - return zng.NewBool(boolean), nil -} - -func (p *inferParser) parseNumber(b []byte) (zng.Value, error) { - d, err := byteconv.ParseFloat64(b) - if err != nil { - return zng.Value{}, err - } - return zng.NewFloat64(d), nil -} - -func (p *inferParser) parseString(b []byte) (zng.Value, error) { - b, err := jsonparser.Unescape(b, nil) - if err != nil { - return zng.Value{}, err - } - s, err := tzngio.ParseString(b) - if err != nil { - return zng.Value{}, err - } - return zng.Value{zng.TypeString, s}, nil -} - -func (p *inferParser) parseNull() (zng.Value, error) { - return zng.Value{zng.TypeString, nil}, nil -} diff --git a/zio/ndjsonio/ndjson_test.go b/zio/ndjsonio/ndjson_test.go index dbefc1d4b9..078883b664 100644 --- a/zio/ndjsonio/ndjson_test.go +++ b/zio/ndjsonio/ndjson_test.go @@ -8,9 +8,7 @@ import ( "testing" "github.com/brimdata/zed/zbuf" - "github.com/brimdata/zed/zng" "github.com/brimdata/zed/zson" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -66,85 +64,6 @@ func TestNDJSONWriter(t *testing.T) { } } -func TestNDJSON(t *testing.T) { - type testcase struct { - name, input, output string - } - cases := []testcase{ - { - name: "single line", - input: `{ "string1": "value1", "int1": 1, "double1": 1.2, "bool1": false }`, - output: "", - }, - { - name: "skips empty lines", - input: `{ "string1": "value1", "int1": 1, "double1": 1.2, "bool1": false } - - {"string1": "value2", "int1": 2, "double1": 2.3, "bool1": true } - `, - output: "", - }, - { - name: "nested containers", - input: `{ "obj1": { "obj2": { "double1": 1.1 } } } - { "arr1": [ "string1", "string2", "string3" ] }`, - output: "", - }, - { - name: "null value", - input: `{ "null1": null }`, - output: "", - }, - { - name: "empty array", - input: `{ "arr1": [] }`, - output: "", - }, - { - name: "legacy nested fields", - input: `{ "s": "foo", "nest.s": "bar", "nest.n": 5 }`, - output: `{ "s": "foo", "nest": { "s": "bar", "n": 5 }}`, - }, - { - name: "legacy nested fields with multiple levels of nesting", - input: `{ "a.b.1": 1, "a.b.2": 2, "a.b.c.3": 3, "a.b.c.4": 4 }`, - output: `{ "a": { "b": { "1": 1, "2": 2, "c": { "3": 3, "4": 4 } } } }`, - }, - { - name: "string with unicode escape", - input: `{ "s": "Hello\u002c world!" }`, - output: `{ "s": "Hello, world!" }`, - }, - // Test that unicode combining characters are properly - // normalized. Note that in the input string, zq interprets - // the \u escapes, while in the output string they are part of - // the go string literal and interpreted by the go compiler. - { - name: "string with unicode combining characters", - input: `{ "s": "E\u0301l escribio\u0301 un caso de prueba"}`, - output: "{ \"s\": \"\u00c9l escribi\u00f3 un caso de prueba\"}", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - output := c.output - if len(output) == 0 { - output = c.input - } - runtestcase(t, c.input, output) - }) - } -} - -func runtestcase(t *testing.T, input, output string) { - var out bytes.Buffer - w := NewWriter(NopCloser(&out)) - r, err := NewReader(strings.NewReader(input), zson.NewContext(), ReaderOpts{}, "") - require.NoError(t, err) - require.NoError(t, zbuf.Copy(w, r)) - NDJSONEq(t, output, out.String()) -} - func getLines(in string) ([]string, error) { var lines []string scanner := bufio.NewScanner(strings.NewReader(in)) @@ -168,205 +87,3 @@ func NDJSONEq(t *testing.T, expected string, actual string) { require.JSONEq(t, expectedLines[i], actualLines[i]) } } - -func TestNewRawFromJSON(t *testing.T) { - type testcase struct { - name, expected, json, defaultPath string - } - cases := []testcase{ - { - name: "LongDuration", - expected: `{_path:"test",ts:2019-11-15T23:30:44.637486Z,span:123.456134ms}`, - json: `{"_path": "test", "ts": "2019-11-15T23:30:44.637486Z", "span": 0.1234561341234234}`, - }, - { - name: "TsISO8601", - expected: `{_path:"test",b:null (bool),i:null (int64),s:null (0=(|[bool]|)),ts:2019-11-15T23:30:44.637486Z,v:null (1=([int64]))}`, - json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486Z"}`, - }, - { - name: "TsISO8601-0100", - expected: `{_path:"test",b:null (bool),i:null (int64),s:null (0=(|[bool]|)),ts:2019-11-16T00:30:44.637486Z,v:null (1=([int64]))}`, - json: `{"_path": "test", "ts":"2019-11-15T23:30:44.637486-0100"}`, - }, - { - name: "TsEpoch", - expected: `{_path:"test",ts:2019-11-15T23:30:44.637486Z}`, - json: `{"_path": "test", "ts":1573860644.637486}`, - }, - { - name: "TsMillis", - expected: `{_path:"test",ts:2019-11-15T23:30:44.637Z}`, - json: `{"_path": "test", "ts":1573860644637}`, - }, - { - name: "defaultPath", - expected: `{_path:"inferred",ts:2019-11-15T23:30:44.637Z}`, - json: `{"ts":1573860644637}`, - defaultPath: "inferred", - }, - { - name: "defaultPath (unused)", - expected: `{_path:"test",ts:2019-11-15T23:30:44.637Z}`, - json: `{"_path": "test", "ts":1573860644637}`, - defaultPath: "inferred", - }, - { - name: "uint64 in scientific notation", - expected: `{_path:"test",datetime:1521835103 (uint64)} (=0)`, - json: `{"_path": "test", "datetime":1.521835103E9}`, - defaultPath: "inferred", - }, - { - name: "int64 in scientific notation", - expected: `{_path:"test",datetime:-1521835103}`, - json: `{"_path": "test", "datetime":-1.521835103E9}`, - defaultPath: "inferred", - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - r := zson.NewReader(strings.NewReader(c.expected), zson.NewContext()) - expected, err := r.Read() - require.NoError(t, err) - typ := zng.TypeRecordOf(expected.Type) - ti := &typeInfo{ - flatDesc: typ, - descriptor: typ, - typedVals: make([]typedVal, len(typ.Columns)), - path: []byte(c.defaultPath), - } - bytes, _, err := ti.newRawFromJSON([]byte(c.json)) - require.NoError(t, err) - rec := zng.NewRecord(typ, bytes) - assert.Equal(t, expected.String(), rec.String()) - }) - } -} - -func TestNDJSONTypeErrors(t *testing.T) { - typeConfig := TypeConfig{ - Descriptors: map[string][]interface{}{ - "http_log": []interface{}{ - map[string]interface{}{ - "name": "_path", - "type": "string", - }, - map[string]interface{}{ - "name": "ts", - "type": "time", - }, - map[string]interface{}{ - "name": "uid", - "type": "bstring", - }, - map[string]interface{}{ - "name": "id", - "type": []interface{}{map[string]interface{}{ - "name": "orig_h", - "type": "ip", - }, - }, - }, - }, - }, - Rules: []Rule{ - Rule{"_path", "http", "http_log"}, - }, - } - - var cases = []struct { - name string - result typeStats - input string - success bool - defaultPath string - }{ - { - name: "Valid", - result: typeStats{}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http"} - {"uid":"CXY9a54W2dLZwzPXf1","ts":"2017-03-24T19:59:24.306076Z","id.orig_h":"10.10.7.65","_path":"http"}`, - success: true, - }, - { - name: "Extra field", - result: typeStats{IncompleteDescriptor: 1, FirstBadLine: 2}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http"} -{"ts":"2017-03-24T19:59:24.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http", "extra_field": 1}`, - success: false, - }, - { - name: "Bad line number", - result: typeStats{BadFormat: 1, FirstBadLine: 2}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http"} -{"hiddents":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http"}`, - success: false, - }, - { - name: "Missing Ts", - result: typeStats{BadFormat: 1, FirstBadLine: 1}, - input: `{"uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65", "_path": "http"}` + "\n", - success: false, - }, - { - name: "Negative Ts", - result: typeStats{BadFormat: 1, FirstBadLine: 1}, - input: `{"ts":"-1579438676.648","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65", "_path": "http"}` + "\n", - success: false, - }, - { - name: "Valid (inferred)", - result: typeStats{DescriptorNotFound: 1, FirstBadLine: 1}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"inferred"}`, - success: false, - }, - { - name: "Missing _path", - result: typeStats{DescriptorNotFound: 1, FirstBadLine: 1}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65"}` + "\n", - success: false, - }, - { - name: "_path provided as defaultPath", - result: typeStats{}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65"}` + "\n", - success: true, - defaultPath: "http", - }, - { - name: "invalid _path provided as defaultPath", - result: typeStats{DescriptorNotFound: 1, FirstBadLine: 1}, - input: `{"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65"}` + "\n", - success: false, - defaultPath: "nosuchpath", - }, - { - name: "invalid defaultPath doesn't override input _path", - result: typeStats{}, - input: `{"_path": "http", "ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65"}` + "\n", - success: true, - defaultPath: "nosuchpath", - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - var out bytes.Buffer - w := NewWriter(NopCloser(&out)) - r, err := NewReader(strings.NewReader(c.input), zson.NewContext(), ReaderOpts{}, "") - require.NoError(t, err) - err = r.configureTypes(typeConfig, c.defaultPath, nil) - require.NoError(t, err) - - err = zbuf.Copy(w, r) - if c.success { - require.NoError(t, err) - } else { - require.Error(t, err) - } - require.Equal(t, c.result, *r.stats.typeStats) - }) - } -} diff --git a/zio/ndjsonio/reader.go b/zio/ndjsonio/reader.go deleted file mode 100644 index 2b71dbe963..0000000000 --- a/zio/ndjsonio/reader.go +++ /dev/null @@ -1,176 +0,0 @@ -// Package ndjsonio parses ndjson records. It can do basic -// transcription of json types into the corresponding zng types, or -// more advanced mapping into zng types using definitions in a -// TypeConfig. -package ndjsonio - -import ( - "bytes" - "fmt" - "io" - "regexp" - - "github.com/brimdata/zed/pkg/skim" - "github.com/brimdata/zed/zio/ndjsonio/compat" - "github.com/brimdata/zed/zio/tzngio" - "github.com/brimdata/zed/zng" - "github.com/brimdata/zed/zson" - "github.com/buger/jsonparser" -) - -const ( - ReadSize = 64 * 1024 - MaxLineSize = 50 * 1024 * 1024 -) - -// x509.14:00:00-15:00:00.log.gz (open source zeek) -// x509_20191101_14:00:00-15:00:00+0000.log.gz (corelight) -const DefaultPathRegexp = `([a-zA-Z0-9_]+)(?:\.|_\d{8}_)\d\d:\d\d:\d\d\-\d\d:\d\d:\d\d(?:[+\-]\d{4})?\.log(?:$|\.gz)` - -type ReaderOpts struct { - TypeConfig *TypeConfig - PathRegexp string - Warnings chan<- string -} - -type ReadStats struct { - *skim.Stats - *typeStats -} - -type Reader struct { - scanner *skim.Scanner - inf inferParser - typ *typeParser - zctx *zson.Context - stats ReadStats - types *tzngio.TypeParser -} - -func NewReader(reader io.Reader, zctx *zson.Context, opts ReaderOpts, filepath string) (*Reader, error) { - // Note: we add hardwired aliases for "port" to "uint16" when reading - // *any* json file but they are only used when the schema mapper - // (aka typings config) references such types from a configured schema. - // However, the schema mapper should be responsible for creating these - // aliases according to its configuration. See issue #1427. - _, err := zctx.LookupTypeAlias("zenum", zng.TypeString) - if err != nil { - return nil, err - } - _, err = zctx.LookupTypeAlias("port", zng.TypeUint16) - if err != nil { - return nil, err - } - buffer := make([]byte, ReadSize) - scanner := skim.NewScanner(reader, buffer, MaxLineSize) - r := &Reader{ - scanner: scanner, - stats: ReadStats{Stats: &scanner.Stats, typeStats: &typeStats{}}, - inf: inferParser{zctx}, - zctx: zctx, - types: tzngio.NewTypeParser(zctx), - } - if opts.TypeConfig != nil { - var path string - re, err := regexp.Compile(opts.PathRegexp) - if err != nil { - return nil, err - } - //XXX why do we do this this way? - match := re.FindStringSubmatch(filepath) - if len(match) == 2 { - path = match[1] - } - if err = r.configureTypes(*opts.TypeConfig, path, opts.Warnings); err != nil { - return nil, err - } - } - return r, nil -} - -// typeRules is used internally and is derived from TypeConfig by -// converting its descriptors into *zng.TypeRecord s for use by the -// ndjson typed parser. -type typeRules struct { - descriptors map[string]*zng.TypeRecord - rules []Rule -} - -// configureTypes adds a TypeConfig to the reader. Its should be -// called before input lines are processed. If a non-empty defaultPath -// is passed, it is used for json objects without a _path. -// In the absence of a TypeConfig, records are all parsed with the -// inferParser. If a TypeConfig is present, records are parsed -// with the typeParser. -func (r *Reader) configureTypes(tc TypeConfig, defaultPath string, warn chan<- string) error { - tr := typeRules{ - descriptors: make(map[string]*zng.TypeRecord), - rules: tc.Rules, - } - - for key, columns := range tc.Descriptors { - typeName, err := compat.DecodeType(columns) - if err != nil { - return fmt.Errorf("error decoding type \"%s\": %s", typeName, err) - } - typ, err := r.types.Parse(typeName) - if err != nil { - return err - } - recType, ok := typ.(*zng.TypeRecord) - if !ok { - return fmt.Errorf("type not a record: \"%s\"", typeName) - } - tr.descriptors[key] = recType - } - r.typ = &typeParser{ - zctx: r.zctx, - tr: tr, - stats: r.stats.typeStats, - typeInfoCache: make(map[int]*typeInfo), - defaultPath: defaultPath, - passUnknowns: tc.PassUnknowns, - warn: warn, - warnSent: make(map[string]struct{}), - } - return nil -} - -// Parse returns a zng.Value from the provided JSON input. The -// function expects the input json to be an object, otherwise an error -// is returned. -func (r *Reader) Parse(b []byte) (zng.Value, error) { - val, typ, _, err := jsonparser.Get(b) - if err != nil { - return zng.Value{}, err - } - if typ != jsonparser.Object { - return zng.Value{}, fmt.Errorf("expected JSON type to be Object but got %s", typ) - } - if r.typ != nil { - return r.typ.parseObject(val, r.inf) - } - return r.inf.parseObject(val) -} - -func (r *Reader) Read() (*zng.Record, error) { -again: - line, err := r.scanner.ScanLine() - if line == nil { - return nil, err - } - line = bytes.TrimSpace(line) - // skip empty lines - if len(line) == 0 { - goto again - } - zv, err := r.Parse(line) - if err != nil { - return nil, fmt.Errorf("line %d: %w", r.scanner.Stats.Lines, err) - } - outType, err := r.zctx.LookupTypeRecord(zv.Type.(*zng.TypeRecord).Columns) - if err != nil { - return nil, err - } - return zng.NewRecordCheck(outType, zv.Bytes) -} diff --git a/zio/ndjsonio/typegenerator/generator.go b/zio/ndjsonio/typegenerator/generator.go deleted file mode 100644 index 41b95b0608..0000000000 --- a/zio/ndjsonio/typegenerator/generator.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "go/format" - "io/ioutil" - "log" - "os" - - "github.com/brimdata/zed/cli/inputflags" -) - -func main() { - var outName, packageName, varName string - - flag.StringVar(&outName, "o", "", "output filename") - flag.StringVar(&packageName, "package", "main", "package name") - flag.StringVar(&varName, "var", "tc", "variable name") - - flag.Parse() - if flag.NArg() == 0 { - flag.Usage() - os.Exit(1) - } - fileName := flag.Arg(0) - tc, err := inputflags.LoadJSONConfig(fileName) - if err != nil { - fmt.Fprintf(os.Stderr, "error parsing json config: %s\n", err) - return - } - - contents := fmt.Sprintf(`package %s - -import "github.com/brimdata/zed/zio/ndjsonio" - -var %s *ndjsonio.TypeConfig = %#v`, packageName, varName, tc) - - formatted, err := format.Source([]byte(contents)) - if err != nil { - fmt.Fprintf(os.Stderr, "error formatting code: %s\n", err) - return - } - - err = ioutil.WriteFile(outName, formatted, 0644) - if err != nil { - log.Fatalf("Error writing to %s: %s\n", outName, err) - } -} diff --git a/zio/ndjsonio/typeparser.go b/zio/ndjsonio/typeparser.go deleted file mode 100644 index b2fea1a105..0000000000 --- a/zio/ndjsonio/typeparser.go +++ /dev/null @@ -1,439 +0,0 @@ -package ndjsonio - -import ( - "bytes" - "errors" - "fmt" - "strings" - "time" - - "github.com/brimdata/zed/pkg/byteconv" - "github.com/brimdata/zed/pkg/nano" - "github.com/brimdata/zed/zcode" - "github.com/brimdata/zed/zio/tzngio" - "github.com/brimdata/zed/zng" - "github.com/brimdata/zed/zng/flattener" - "github.com/brimdata/zed/zson" - "github.com/buger/jsonparser" -) - -type typeStats struct { - BadFormat int - FirstBadLine int - DescriptorNotFound int - IncompleteDescriptor int -} - -type typeParser struct { - lineNo int - zctx *zson.Context - tr typeRules - defaultPath string - stats *typeStats - typeInfoCache map[int]*typeInfo - passUnknowns bool - warn chan<- string - warnSent map[string]struct{} -} - -var ( - ErrDescriptorNotFound = errors.New("descriptor not found") - ErrIncompleteDescriptor = errors.New("incomplete descriptor") -) - -// Information about the correspondence between the flattened structure -// of a JSON object and its zng representation (which may include -// nested record fields). The two descriptors here represent the same data -// in the same order, flatDescriptor describes the data as it appears in -// JSON, descriptor describes it as it appears in zng values. -type typeInfo struct { - descriptor *zng.TypeRecord - flatDesc *zng.TypeRecord - path []byte - typedVals []typedVal - untypedVals []untypedVal - zctx *zson.Context -} -type typedVal struct { - val []byte - typ jsonparser.ValueType -} - -type untypedVal struct { - key []byte - val []byte - typ jsonparser.ValueType -} - -func getUnsafeDefault(data []byte, defaultValue string, key string) (string, error) { - val, err := jsonparser.GetUnsafeString(data, key) - if err != nil { - // This is always a KeyPathNotFoundError, including if the json was invalid. - if defaultValue == "" { - return "", jsonparser.KeyPathNotFoundError - } - return defaultValue, nil - } - return val, nil -} - -func newTypeInfo(zctx *zson.Context, desc *zng.TypeRecord, path string) (*typeInfo, error) { - flatCols := flattener.FlattenColumns(desc.Columns) - flatDesc, err := zctx.LookupTypeRecord(flatCols) - if err != nil { - return nil, err - } - info := typeInfo{desc, flatDesc, []byte(path), make([]typedVal, len(flatDesc.Columns)), make([]untypedVal, 0, len(flatDesc.Columns)), zctx} - return &info, nil -} - -func (info *typeInfo) makeViews(data []byte) ([]untypedVal, error) { - var droppedFields int - - for i := range info.typedVals { - info.typedVals[i].typ = jsonparser.NotExist - } - info.untypedVals = info.untypedVals[:0] - - // path is always the first field (typings config is validated - // for this, and inferred TDs are sorted with _path first). - info.typedVals[0] = typedVal{info.path, jsonparser.String} - - var prefix []string - - // callback can't be declared in one line due to golang/go#226 - var callback func(key []byte, val []byte, typ jsonparser.ValueType, offset int) error - callback = func(key []byte, val []byte, typ jsonparser.ValueType, offset int) error { - skey := string(key) - if typ == jsonparser.Object { - prefix = append(prefix, skey) - err := jsonparser.ObjectEach(val, callback) - prefix = prefix[0 : len(prefix)-1] - return err - } - - fullkey := strings.Join(append(prefix, skey), ".") - - if col, ok := info.flatDesc.ColumnOfField(fullkey); ok { - info.typedVals[col] = typedVal{val, typ} - } else { - info.untypedVals = append(info.untypedVals, untypedVal{[]byte(fullkey), val, typ}) - droppedFields++ - } - return nil - } - if err := jsonparser.ObjectEach(data, callback); err != nil { - return nil, err - } - return info.untypedVals, nil -} - -func appendRecordFromViews(builder *zcode.Builder, columns []zng.Column, typedVals []typedVal) ([]typedVal, error) { - handleVal := func(jv typedVal, col zng.Column) error { - switch jv.typ { - case jsonparser.Array: - builder.BeginContainer() - ztyp := zng.InnerType(col.Type) - if ztyp == nil { - return zng.ErrNotPrimitive - } - var iterErr error - callback := func(v []byte, typ jsonparser.ValueType, offset int, _ error) { - zv, err := parseSimpleType(v, ztyp) - if err != nil { - iterErr = fmt.Errorf("field \"%s\" (type %s): %w", col.Name, typ, err) - } else { - builder.AppendPrimitive(zv) - } - } - if _, err := jsonparser.ArrayEach(jv.val, callback); err != nil { - return err - } - if iterErr != nil { - return iterErr - } - if _, ok := col.Type.(*zng.TypeSet); ok { - builder.TransformContainer(zng.NormalizeSet) - } - builder.EndContainer() - case jsonparser.NotExist, jsonparser.Null: - switch col.Type.(type) { - case *zng.TypeSet, *zng.TypeArray: - builder.AppendContainer(nil) - default: - builder.AppendPrimitive(nil) - } - default: - zv, err := parseSimpleType(jv.val, col.Type) - if err != nil { - return fmt.Errorf("field \"%s\" (type %s): %w", col.Name, col.Type, err) - } - builder.AppendPrimitive(zv) - } - return nil - } - - c := 0 - for c < len(columns) { - if len(typedVals) == 0 { - return nil, errors.New("too few values") - } - - typ := columns[c].Type - if recType, isRec := typ.(*zng.TypeRecord); isRec { - builder.BeginContainer() - var err error - if typedVals, err = appendRecordFromViews(builder, recType.Columns, typedVals); err != nil { - return nil, err - } - builder.EndContainer() - } else { - if err := handleVal(typedVals[0], columns[c]); err != nil { - return nil, err - } - typedVals = typedVals[1:] - } - c++ - } - return typedVals, nil -} - -// newRawFromJSON builds a raw value from a descriptor and the JSON object -// in data. It works in two steps. First, it constructs a slice of views onto -// the underlying JSON values. This slice follows the order of the flattened -// columns. Second, it builds the full encoded value and building nested -// records as necessary. -func (info *typeInfo) newRawFromJSON(data []byte) (zcode.Bytes, []untypedVal, error) { - - toInfer, err := info.makeViews(data) - if err != nil { - return nil, nil, err - } - - i, ok := info.descriptor.ColumnOfField("ts") - if ok && info.typedVals[i].typ != jsonparser.String && info.typedVals[i].typ != jsonparser.Number { - return nil, nil, fmt.Errorf("invalid json type for ts: %s", info.typedVals[i].typ) - } - - builder := zcode.NewBuilder() - - _, err = appendRecordFromViews(builder, info.descriptor.Columns, info.typedVals) - if err != nil { - return nil, nil, err - } - return builder.Bytes(), toInfer, nil -} - -// findTypeInfo returns the typeInfo struct matching an input json -// object. If no match is found, an error is returned. If defaultPath -// is not empty, it is used as the default _path if the object has no -// such field. (we could at some point make this a bit more generic by -// passing in a "defaultFieldValues" map... but not needed now). -func (p *typeParser) findTypeInfo(zctx *zson.Context, jobj []byte, tr typeRules, defaultPath string) (*typeInfo, error) { - var fieldName, fieldVal, path string - for _, r := range tr.rules { - // we keep track of the last field value we extracted - // to avoid re-parsing the json object many times to - // lift out the same field, as would be the case with - // a typical zeek typing config where all rules refer - // to the field "_path". - if fieldName != r.Name { - fieldName = r.Name - var err error - if r.Name == "_path" { - fieldVal, err = getUnsafeDefault(jobj, defaultPath, r.Name) - path = fieldVal - } else { - // jsonparser.Get will return the key even for - // some invalid json. For example Get('x{"a": - // "b"}', "a") returns "b". This is ok because - // these errors will later be caught by ObjectEach. - fieldVal, err = jsonparser.GetUnsafeString(jobj, r.Name) - } - if err != nil { - continue - } - } - if fieldVal == r.Value { - desc := tr.descriptors[r.Descriptor] - if ti, ok := p.typeInfoCache[desc.ID()]; ok { - return ti, nil - } - ti, err := newTypeInfo(zctx, desc, path) - if err != nil { - return nil, err - } - p.typeInfoCache[desc.ID()] = ti - return ti, nil - } - } - return nil, ErrDescriptorNotFound -} - -func (p *typeParser) parseObject(b []byte, inferrer inferParser) (zng.Value, error) { - incr := func(stat *int) { - (*stat)++ - if p.stats.FirstBadLine == 0 { - p.stats.FirstBadLine = p.lineNo - } - } - - p.lineNo++ - ti, err := p.findTypeInfo(p.zctx, b, p.tr, p.defaultPath) - if err != nil { - switch err { - case ErrDescriptorNotFound: - incr(&p.stats.DescriptorNotFound) - default: - panic("unhandled error") - } - return zng.Value{}, err - } - - raw, toInfer, err := ti.newRawFromJSON(b) - if err != nil { - incr(&p.stats.BadFormat) - return zng.Value{}, err - } - if len(toInfer) > 0 { - if !p.passUnknowns { - incr(&p.stats.IncompleteDescriptor) - return zng.Value{}, ErrIncompleteDescriptor - } - if p.warn != nil { - msg := "Unexpected additional field(s)" - for _, v := range toInfer { - msg = msg + " '" + string(v.key) + "'" - } - msg = msg + ". Please email this error message to support@brimsecurity.com." - if _, ok := p.warnSent[msg]; !ok { - p.warn <- msg - p.warnSent[msg] = struct{}{} - } - } - cols := ti.descriptor.Columns - for _, v := range toInfer { - zval, err := inferrer.parseValue(v.val, v.typ) - if err != nil { - return zng.Value{}, fmt.Errorf("invalid json for inferred value: %s", string(v.val)) - } - raw = zcode.AppendAs(raw, zng.IsContainerType(zval.Type), zval.Bytes) - cols = append(cols, zng.Column{Name: string(v.key), Type: zval.Type}) - } - tr, err := p.zctx.LookupTypeRecord(cols) - if err != nil { - return zng.Value{}, err - } - return zng.Value{tr, raw}, nil - } - return zng.Value{ti.descriptor, raw}, nil -} - -func parseSimpleType(value []byte, typ zng.Type) ([]byte, error) { - if zng.IsContainerType(typ) { - return nil, zng.ErrNotContainer - } - switch typ { - case zng.TypeTime: - ts, err := parseJSONTimestamp(value) - if err != nil { - return nil, err - } - return zng.EncodeTime(ts), nil - case zng.TypeDuration: - // cannot use nano.Parse because javascript floats values can have - // greater precision than 1e-9. - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeInt(int64(f * 1e9)), nil - case zng.TypeUint64: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeUint(uint64(f)), nil - case zng.TypeInt64: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeInt(int64(f)), nil - case zng.TypeUint32: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeUint(uint64(uint32(f))), nil - case zng.TypeInt32: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeInt(int64(int32(f))), nil - case zng.TypeUint16: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeUint(uint64(uint16(f))), nil - case zng.TypeInt16: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeInt(int64(int16(f))), nil - case zng.TypeUint8: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeUint(uint64(uint8(f))), nil - case zng.TypeInt8: - f, err := byteconv.ParseFloat64(value) - if err != nil { - return nil, err - } - return zng.EncodeInt(int64(int8(f))), nil - default: - b, err := tzngio.ParseValue(typ, value) - if err != nil { - return nil, err - } - return b, nil - } -} - -func parseISO8601SignedOffset(s []byte) (nano.Ts, error) { - t, err := time.Parse("2006-01-02T15:04:05.999999999-0700", string(s)) - if err != nil { - return 0, err - } - return nano.TimeToTs(t), nil -} - -// parseJSONTimestamp interprets data as a timestamp and returns its value as -// both a nano.Ts and the standard Zeek format (a decimal floating-point number -// representing seconds since the Unix epoch). -// -// parseJSONTimestamp understands the three timestamp formats that -// Zeek's ASCII log writer can produce when LogAscii::use_json is true -// as well as the ISO8601 format emitted by Suricata in eve.json. -// -// The Zeek formats correspond to the three possible values for -// LogAscii::json_timestamps: JSON::TS_EPOCH, JSON::TS_ISO8601, and -// JSON::TS_MILLIS. For descriptions, see -// https://docs.zeek.org/en/stable/scripts/base/init-bare.zeek.html#type-JSON::TimestampFormat. -func parseJSONTimestamp(data []byte) (nano.Ts, error) { - switch { - case bytes.Contains(data, []byte{'Z'}): // Zeek JSON::TS_ISO8601 - return nano.ParseRFC3339Nano(data) - case bytes.Contains(data, []byte{'-'}): - return parseISO8601SignedOffset(data) - case bytes.Contains(data, []byte{'.'}): // Zeek JSON::TS_EPOCH - return nano.Parse(data) - default: // Zeek JSON::TS_MILLIS - return nano.ParseMillis(data) - } -} diff --git a/zio/ndjsonio/ztests/error-descriptor-not-found.yaml b/zio/ndjsonio/ztests/error-descriptor-not-found.yaml deleted file mode 100644 index b87df4a17e..0000000000 --- a/zio/ndjsonio/ztests/error-descriptor-not-found.yaml +++ /dev/null @@ -1,37 +0,0 @@ -script: | - zq -i ndjson -j types.json "*" *.ndjson - -inputs: - - name: http.ndjson - data: | - {"ts":"2017-03-24T19:59:23.306076Z","_path":"http"} - - name: badpath.ndjson - data: | - {"ts":"2017-03-24T19:59:23.306076Z","_path":"badpath"} - - name: types.json - data: | - { - "descriptors": { - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "http", - "descriptor": "http_log" - } - ] - } - -outputs: - - name: stderr - regexp: .*descriptor not found.* diff --git a/zio/ndjsonio/ztests/types-file-autodetect-nonjson.yaml b/zio/ndjsonio/ztests/types-file-autodetect-nonjson.yaml deleted file mode 100644 index a1be346a8c..0000000000 --- a/zio/ndjsonio/ztests/types-file-autodetect-nonjson.yaml +++ /dev/null @@ -1,18 +0,0 @@ -script: zq -z -j types.json "*" in.zng - -inputs: - - name: in.zng - data: !!binary 9gEBcxAXAgRhFwIEYhcCBGH/ - - name: types.json - data: | - { - "descriptors": {}, - "rules": [] - } - -outputs: - - name: stdout - data: | - {s:"a"} - {s:"b"} - {s:"a"} diff --git a/zio/ndjsonio/ztests/types-file-badconfig.yaml b/zio/ndjsonio/ztests/types-file-badconfig.yaml deleted file mode 100644 index a469729cf5..0000000000 --- a/zio/ndjsonio/ztests/types-file-badconfig.yaml +++ /dev/null @@ -1,35 +0,0 @@ -script: zq -i ndjson -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"_path":"simple", "number": 4} - - name: types.json - data: | - { - "descriptors": { - "simple_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "number", - "type": "nosuchtype" - } - ] - }, - "rules": [ - { - "descriptor": "simple_log", - "name": "_path", - "value": "simple" - } - ] - } - - -outputs: - - name: stderr - regexp: | - unknown type: nosuchtype diff --git a/zio/ndjsonio/ztests/types-file-errnotcontainer.yaml b/zio/ndjsonio/ztests/types-file-errnotcontainer.yaml deleted file mode 100644 index 50ff6d4597..0000000000 --- a/zio/ndjsonio/ztests/types-file-errnotcontainer.yaml +++ /dev/null @@ -1,36 +0,0 @@ -script: zq -i ndjson -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"_path":"test","f": 1} - - name: types.json - data: | - { - "descriptors": { - "test_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "f", - "type": "array[int64]" - } - ] - }, - "rules": [ - { - "descriptor": "test_log", - "name": "_path", - "value": "test" - } - ] - } - - - -outputs: - - name: stderr - regexp: | - field "f" \(type array\[int64\]\): expected container type, got primitive diff --git a/zio/ndjsonio/ztests/types-file-errnotprimitive.yaml b/zio/ndjsonio/ztests/types-file-errnotprimitive.yaml deleted file mode 100644 index fc350fa7ed..0000000000 --- a/zio/ndjsonio/ztests/types-file-errnotprimitive.yaml +++ /dev/null @@ -1,36 +0,0 @@ -script: zq -i ndjson -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"_path":"test","f": [1]} - - name: types.json - data: | - { - "descriptors": { - "test_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "f", - "type": "int64" - } - ] - }, - "rules": [ - { - "descriptor": "test_log", - "name": "_path", - "value": "test" - } - ] - } - - - -outputs: - - name: stderr - regexp: | - expected primitive type, got container diff --git a/zio/ndjsonio/ztests/types-file-errormsg.yaml b/zio/ndjsonio/ztests/types-file-errormsg.yaml deleted file mode 100644 index 76e24f4fbd..0000000000 --- a/zio/ndjsonio/ztests/types-file-errormsg.yaml +++ /dev/null @@ -1,35 +0,0 @@ -script: zq -i ndjson -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"_path":"dns","datetime":"foo"} - - name: types.json - data: | - { - "descriptors": { - "dns_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "datetime", - "type": "int64" - } - ] - }, - "rules": [ - { - "descriptor": "dns_log", - "name": "_path", - "value": "dns" - } - ] - } - - -outputs: - - name: stderr - regexp: | - field "datetime" \(type int64\): strconv.ParseFloat: parsing "foo": invalid syntax diff --git a/zio/ndjsonio/ztests/types-file-nesting.yaml b/zio/ndjsonio/ztests/types-file-nesting.yaml deleted file mode 100644 index 12dff4c88a..0000000000 --- a/zio/ndjsonio/ztests/types-file-nesting.yaml +++ /dev/null @@ -1,58 +0,0 @@ -script: zq -z -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"_path":"http", "l1.l2.l3a": "deep1", "ts":"1521911721.255387", "l1.l2.l3b": "deep2"} - {"_path":"http", "ts":"1521911721.255387", "l1.l2.l3a": "deep1"} - {"_path":"http", "ts":"1521911721.255387", "l1.l2.l3a": "deep1", "l1.l2.l3b": null} - {"_path":"http", "ts":"1521911721.255387"} - - name: types.json - data: | - { - "descriptors": { - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "l1", - "type": [ - { - "name": "l2", - "type": [ - { - "name": "l3a", - "type": "string" - }, - { - "name": "l3b", - "type": "string" - } - ] - } - ] - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "http", - "descriptor": "http_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"http",ts:2018-03-24T17:15:21.255387Z,l1:{l2:{l3a:"deep1",l3b:"deep2"}}} - {_path:"http",ts:2018-03-24T17:15:21.255387Z,l1:{l2:{l3a:"deep1",l3b:null (string)}}} - {_path:"http",ts:2018-03-24T17:15:21.255387Z,l1:{l2:{l3a:"deep1",l3b:null (string)}}} - {_path:"http",ts:2018-03-24T17:15:21.255387Z,l1:{l2:{l3a:null (string),l3b:null (string)}}} diff --git a/zio/ndjsonio/ztests/types-file-passunknowns.yaml b/zio/ndjsonio/ztests/types-file-passunknowns.yaml deleted file mode 100644 index 07bf0c29eb..0000000000 --- a/zio/ndjsonio/ztests/types-file-passunknowns.yaml +++ /dev/null @@ -1,57 +0,0 @@ -script: zq -z -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"ts":"2017-03-24T19:59:23.306076Z","id.orig_h":"1.1.1.1","id.other_h":"1.1.1.2", "id2": {"orig_h": "2.2.2.1", "other_h": "2.2.2.2"}, "id3": {"orig_h": "2.2.2.1"}, "_path":"http", "uid":"CXY9a54W2dLZwzPXf1"} - - name: types.json - data: | - { - "pass_unknowns": true, - "descriptors": { - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - } - ] - }, - { - "name": "id2", - "type": [ - { - "name": "orig_h", - "type": "ip" - } - ] - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "http", - "descriptor": "http_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"http",ts:2017-03-24T19:59:23.306076Z,uid:"CXY9a54W2dLZwzPXf1" (bstring),id:{orig_h:1.1.1.1},id2:{orig_h:2.2.2.1},"id.other_h":"1.1.1.2","id2.other_h":"2.2.2.2","id3.orig_h":"2.2.2.1"} (=0) diff --git a/zio/ndjsonio/ztests/types-file-test-infer-path.yaml b/zio/ndjsonio/ztests/types-file-test-infer-path.yaml deleted file mode 100644 index dcad598a94..0000000000 --- a/zio/ndjsonio/ztests/types-file-test-infer-path.yaml +++ /dev/null @@ -1,47 +0,0 @@ -script: zq -z -j types.json "*" *.log - -inputs: - - name: http_20190830_08:00:00-09:00:00-0500.log - data: | - {"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65"} - - name: types.json - data: | - { - "descriptors": { - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - } - ] - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "http", - "descriptor": "http_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"http",ts:2017-03-24T19:59:23.306076Z,uid:"CXY9a54W2dLZwzPXf1" (bstring),id:{orig_h:10.10.7.65}} (=0) diff --git a/zio/ndjsonio/ztests/types-file-test-no-ts.yaml b/zio/ndjsonio/ztests/types-file-test-no-ts.yaml deleted file mode 100644 index 606a7ce8e4..0000000000 --- a/zio/ndjsonio/ztests/types-file-test-no-ts.yaml +++ /dev/null @@ -1,34 +0,0 @@ -script: zq -z -j types.json in.ndjson - -inputs: - - name: in.ndjson - data: | - {"name": "foo","_path":"nots"} - - name: types.json - data: | - { - "descriptors": { - "nots_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "name", - "type": "bstring" - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "nots", - "descriptor": "nots_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"nots",name:"foo" (bstring)} (=0) diff --git a/zio/ndjsonio/ztests/types-file-test-set.yaml b/zio/ndjsonio/ztests/types-file-test-set.yaml deleted file mode 100644 index d90271e76b..0000000000 --- a/zio/ndjsonio/ztests/types-file-test-set.yaml +++ /dev/null @@ -1,38 +0,0 @@ -script: zq -z -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"ts":"2017-03-24T19:59:23.306076Z","uids":["b", "a"],"_path":"sets"} - - name: types.json - data: | - { - "descriptors": { - "sets_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uids", - "type": "set[bstring]" - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "sets", - "descriptor": "sets_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"sets",ts:2017-03-24T19:59:23.306076Z,uids:|["a" (bstring),"b" (bstring)]| (=0)} (=1) diff --git a/zio/ndjsonio/ztests/types-file-test-ts.yaml b/zio/ndjsonio/ztests/types-file-test-ts.yaml deleted file mode 100644 index 38040c0806..0000000000 --- a/zio/ndjsonio/ztests/types-file-test-ts.yaml +++ /dev/null @@ -1,34 +0,0 @@ -script: zq -z -j types.json "every 24h count()" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"ts":"2015-03-05T14:25:14.419939Z","_path":"ts"} - - name: types.json - data: | - { - "descriptors": { - "ts_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "ts", - "descriptor": "ts_log" - } - ] - } - -outputs: - - name: stdout - data: | - {ts:2015-03-05T00:00:00Z,count:1 (uint64)} (=0) diff --git a/zio/ndjsonio/ztests/types-file-test.yaml b/zio/ndjsonio/ztests/types-file-test.yaml deleted file mode 100644 index f4611d83fc..0000000000 --- a/zio/ndjsonio/ztests/types-file-test.yaml +++ /dev/null @@ -1,47 +0,0 @@ -script: zq -z -j types.json "*" in.ndjson - -inputs: - - name: in.ndjson - data: | - {"ts":"2017-03-24T19:59:23.306076Z","uid":"CXY9a54W2dLZwzPXf1","id.orig_h":"10.10.7.65","_path":"http"} - - name: types.json - data: | - { - "descriptors": { - "http_log": [ - { - "name": "_path", - "type": "string" - }, - { - "name": "ts", - "type": "time" - }, - { - "name": "uid", - "type": "bstring" - }, - { - "name": "id", - "type": [ - { - "name": "orig_h", - "type": "ip" - } - ] - } - ] - }, - "rules": [ - { - "name": "_path", - "value": "http", - "descriptor": "http_log" - } - ] - } - -outputs: - - name: stdout - data: | - {_path:"http",ts:2017-03-24T19:59:23.306076Z,uid:"CXY9a54W2dLZwzPXf1" (bstring),id:{orig_h:10.10.7.65}} (=0) diff --git a/zio/zio.go b/zio/zio.go index c8b2f4e1b8..553c93404b 100644 --- a/zio/zio.go +++ b/zio/zio.go @@ -4,7 +4,6 @@ import ( "io" "github.com/aws/aws-sdk-go/aws" - "github.com/brimdata/zed/zio/ndjsonio" "github.com/brimdata/zed/zio/textio" "github.com/brimdata/zed/zio/zngio" "github.com/brimdata/zed/zio/zsonio" @@ -14,7 +13,6 @@ import ( type ReaderOpts struct { Format string Zng zngio.ReaderOpts - JSON ndjsonio.ReaderOpts AwsCfg *aws.Config } diff --git a/zng/ztests/issue-2366.yaml b/zng/ztests/issue-2366.yaml deleted file mode 100644 index ea885dc1ee..0000000000 --- a/zng/ztests/issue-2366.yaml +++ /dev/null @@ -1,17 +0,0 @@ -script: | - zq -z -j types.json "count()" sample.ndjson - -inputs: - - name: types.json - source: ../../zeek/types.json - - name: sample.ndjson - data: | - {"_path":"capture_loss","_write_ts":"2020-02-25T16:03:17.838527Z","ts":"2020-02-25T16:03:17.838527Z","ts_delta":11.854892015457153,"peer":"zeek","gaps":0,"acks":6,"percent_lost":0.0} - {"_path":"conn","_write_ts":"2020-02-25T16:03:17.838527Z","ts":"2020-02-25T16:03:11.275550Z","uid":"CQ137z1tDVuWxqRxR2","id.orig_h":"192.168.1.110","id.orig_p":54375,"id.resp_h":"192.168.1.254","id.resp_p":53,"proto":"udp","service":"dns","duration":0.01794600486755371,"orig_bytes":46,"resp_bytes":62,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":74,"resp_pkts":1,"resp_ip_bytes":90} - -outputs: - - name: stdout - data: | - {count:2 (uint64)} (=0) - - name: stderr - data: ""