remove zio/ndjsonio.Reader in favor of zson.Reader

api/api.go

@@ -12,7 +12,6 @@ import (
 
  "github.com/brimdata/zed/pkg/iosrc"
  "github.com/brimdata/zed/pkg/nano"
- "github.com/brimdata/zed/zio/ndjsonio"
  "github.com/brimdata/zed/zio/zjsonio"
 )
 
@@ -162,10 +161,9 @@ type PcapPostStatus struct {
 }
 
 type LogPostRequest struct {
- Paths []string `json:"paths"`
- StopErr bool `json:"stop_err"`
- JSONTypeConfig *ndjsonio.TypeConfig `json:"json_type_config"`
- Shaper json.RawMessage `json:"shaper,omitempty"`
+ Paths []string `json:"paths"`
+ StopErr bool `json:"stop_err"`
+ Shaper json.RawMessage `json:"shaper,omitempty"`
 }
 
 type LogPostWarning struct {

api/client/connection.go

@@ -16,7 +16,6 @@ import (
  "github.com/brimdata/zed/api"
  "github.com/brimdata/zed/compiler/ast"
  "github.com/brimdata/zed/pcap/pcapio"
- "github.com/brimdata/zed/zio/ndjsonio"
  "github.com/brimdata/zed/zqe"
  "github.com/go-resty/resty/v2"
 )
@@ -398,7 +397,6 @@ type PcapReadCloser struct {
 }
 
 type LogPostOpts struct {
- JSON *ndjsonio.TypeConfig
  StopError bool
  Shaper ast.Proc
 }
@@ -417,9 +415,8 @@ func (c *Connection) LogPostPath(ctx context.Context, space api.SpaceID, opts *L
 
 func (c *Connection) LogPostPathStream(ctx context.Context, space api.SpaceID, opts *LogPostOpts, paths ...string) (*Stream, error) {
  body := api.LogPostRequest{
- Paths: paths,
- JSONTypeConfig: opts.JSON,
- StopErr: opts.StopError,
+ Paths: paths,
+ StopErr: opts.StopError,
  }
  if opts != nil && opts.Shaper != nil {
  raw, err := json.Marshal(opts.Shaper)
@@ -468,9 +465,6 @@ func (c *Connection) LogPostWriter(ctx context.Context, space api.SpaceID, opts
  if opts.StopError {
  req.SetQueryParam("stop_err", "true")
  }
- if opts.JSON != nil {
- writer.SetJSONConfig(opts.JSON)
- }
  }
  u := path.Join("/space", url.PathEscape(string(space)), "log")
  resp, err := req.Post(u)

api/client/multipartwriter.go

@@ -12,7 +12,6 @@ import (
 
  "github.com/brimdata/zed/compiler/ast"
  "github.com/brimdata/zed/pkg/iosrc"
- "github.com/brimdata/zed/zio/ndjsonio"
 )
 
 type MultipartWriter struct {
@@ -25,7 +24,6 @@ type MultipartWriter struct {
  start sync.Once
  readers []io.Reader
  uris []iosrc.URI
- json *ndjsonio.TypeConfig
  shaper ast.Proc
 }
 
@@ -60,10 +58,6 @@ func MultipartDataWriter(readers ...io.Reader) (*MultipartWriter, error) {
  return m, nil
 }
 
-func (m *MultipartWriter) SetJSONConfig(config *ndjsonio.TypeConfig) {
- m.json = config
-}
-
 func (m *MultipartWriter) SetShaper(shaper ast.Proc) {
  m.shaper = shaper
 }
@@ -84,10 +78,6 @@ func (m *MultipartWriter) Read(b []byte) (int, error) {
 }
 
 func (m *MultipartWriter) run() {
- if err := m.sendJSONConfig(); err != nil {
- m.pw.CloseWithError(err)
- return
- }
  if err := m.sendShaperAST(); err != nil {
  m.pw.CloseWithError(err)
  return
@@ -126,17 +116,6 @@ func (m *MultipartWriter) write(name string, r io.Reader) error {
  return err
 }
 
-func (m *MultipartWriter) sendJSONConfig() error {
- if m.json == nil {
- return nil
- }
- w, err := m.form.CreateFormField("json_config")
- if err != nil {
- return err
- }
- return json.NewEncoder(w).Encode(m.json)
-}
-
 func (m *MultipartWriter) sendShaperAST() error {
  if m.shaper == nil {
  return nil

cli/inputflags/flags.go

@@ -1,20 +1,16 @@
 package inputflags
 
 import (
- "encoding/json"
  "errors"
  "flag"
  "fmt"
- "io/ioutil"
  "os"
- "regexp"
 
  "github.com/brimdata/zed/cli/auto"
  "github.com/brimdata/zed/pkg/iosrc"
  "github.com/brimdata/zed/zbuf"
  "github.com/brimdata/zed/zio"
  "github.com/brimdata/zed/zio/detector"
- "github.com/brimdata/zed/zio/ndjsonio"
  "github.com/brimdata/zed/zio/zngio"
  "github.com/brimdata/zed/zson"
 )
@@ -23,8 +19,6 @@ type Flags struct {
  zio.ReaderOpts
  ReadMax auto.Bytes
  ReadSize auto.Bytes
- // The JSON type config is loaded from the types filie when Init is called.
- jsonTypesFile string
 }
 
 func (f *Flags) Options() zio.ReaderOpts {
@@ -34,9 +28,6 @@ func (f *Flags) Options() zio.ReaderOpts {
 func (f *Flags) SetFlags(fs *flag.FlagSet) {
  fs.StringVar(&f.Format, "i", "auto", "format of input data [auto,zng,zst,json,ndjson,zeek,zjson,csv,tzng,parquet]")
  fs.BoolVar(&f.Zng.Validate, "validate", true, "validate the input format when reading ZNG streams")
- fs.StringVar(&f.jsonTypesFile, "j", "", "path to json types file")
- fs.StringVar(&f.JSON.PathRegexp, "pathregexp", ndjsonio.DefaultPathRegexp,
- "regexp for extracting _path from json log name (when -inferpath=true)")
  f.ReadMax = auto.NewBytes(zngio.MaxSize)
  fs.Var(&f.ReadMax, "readmax", "maximum memory used read buffers in MiB, MB, etc")
  f.ReadSize = auto.NewBytes(zngio.ReadSize)
@@ -45,17 +36,6 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) {
 
 // Init is called after flags have been parsed.
 func (f *Flags) Init() error {
- // Catch errors early ... ?! XXX
- if _, err := regexp.Compile(f.JSON.PathRegexp); err != nil {
- return err
- }
- if f.jsonTypesFile != "" {
- c, err := LoadJSONConfig(f.jsonTypesFile)
- if err != nil {
- return err
- }
- f.JSON.TypeConfig = c
- }
  f.Zng.Max = int(f.ReadMax.Bytes)
  if f.Zng.Max < 0 {
  return errors.New("max read buffer size must be greater than zero")
@@ -67,21 +47,6 @@ func (f *Flags) Init() error {
  return nil
 }
 
-func LoadJSONConfig(path string) (*ndjsonio.TypeConfig, error) {
- data, err := ioutil.ReadFile(path)
- if err != nil {
- return nil, err
- }
- var tc ndjsonio.TypeConfig
- if err := json.Unmarshal(data, &tc); err != nil {
- return nil, fmt.Errorf("%s: unmarshaling error: %s", path, err)
- }
- if err := tc.Validate(); err != nil {
- return nil, fmt.Errorf("%s: %s", path, err)
- }
- return &tc, nil
-}
-
 func (f *Flags) Open(zctx *zson.Context, paths []string, stopOnErr bool) ([]zbuf.Reader, error) {
  var readers []zbuf.Reader
  for _, path := range paths {

go.mod

@@ -8,7 +8,6 @@ require (
  github.com/araddon/dateparse v0.0.0-20210207001429-0eec95c9db7e
  github.com/aws/aws-sdk-go v1.36.17
  github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f
- github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e
  github.com/dgrijalva/jwt-go v3.2.0+incompatible
  github.com/fraugster/parquet-go v0.3.0
  github.com/fsnotify/fsnotify v1.4.9

go.sum

@@ -69,8 +69,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/brimdata/parquet-go v0.3.1 h1:px/EuItj2/lWMJDcx9H412f4Fl1CNdZSHEtdvniOYq4=
 github.com/brimdata/parquet-go v0.3.1/go.mod h1:qIL8Wm6AK06QHCj9OBFW6PyS+7ukZxc20K/acSeGUas=
-github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e h1:oJCXMss/3rg5F6Poy9wG3JQusc58Mzk5B9Z6wSnssNE=
-github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e/go.mod h1:errmMKH8tTB49UR2A8C8DPYkyudelsYJwJFaZHQ6ik8=
 github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

ppl/zqd/handlers.go

@@ -657,7 +657,7 @@ func handleIntakePostData(c *Core, w http.ResponseWriter, r *http.Request) {
  return
  }
  zctx := zson.NewContext()
- zr, err := detector.NewReaderWithOpts(r.Body, zctx, "", zio.ReaderOpts{Zng: zngio.ReaderOpts{Validate: true}})
+ zr, err := detector.NewReaderWithOpts(r.Body, zctx, zio.ReaderOpts{Zng: zngio.ReaderOpts{Validate: true}})
  if err != nil {
  respondError(c, w, r, err)
  return