-
Notifications
You must be signed in to change notification settings - Fork 67
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add vector compiler skeleton and basic logic (#5013)
This commit adds basic comparison logic to the vector runtime and some initial compiler framework to translate vectorized DAGs more generally into runtime ops and exprs. This is partially complete but we want to merge as is so the team can begin work on vector-enabled field search in parallel. We also renamed the "zed dev vcache" command to "zed dev vector". In a subsequent PR, we will finish out the mechanisms for handling heterogeneously typed data with mixed-in errors building up a library of helper functions for Zed variants to do so. At that point, we will begin adding comprehensive tests for the vector runtime.
- Loading branch information
Showing
39 changed files
with
1,981 additions
and
98 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package query | ||
|
|
||
| import ( | ||
| "errors" | ||
| "flag" | ||
|
|
||
| "github.com/brimdata/zed" | ||
| "github.com/brimdata/zed/cli/outputflags" | ||
| "github.com/brimdata/zed/cmd/zed/dev/vector" | ||
| "github.com/brimdata/zed/cmd/zed/root" | ||
| "github.com/brimdata/zed/compiler" | ||
| "github.com/brimdata/zed/pkg/charm" | ||
| "github.com/brimdata/zed/pkg/storage" | ||
| "github.com/brimdata/zed/runtime" | ||
| "github.com/brimdata/zed/runtime/vcache" | ||
| "github.com/brimdata/zed/zbuf" | ||
| "github.com/brimdata/zed/zio" | ||
| "github.com/segmentio/ksuid" | ||
| ) | ||
|
|
||
| var query = &charm.Spec{ | ||
| Name: "query", | ||
| Usage: "query [flags] query path", | ||
| Short: "run a Zed query on a VNG file", | ||
| Long: ` | ||
| The query command runs a query on a VNG file presuming the | ||
| query is entirely vectorizable. The VNG object is read through | ||
| the vcache and projected as needed into the runtime. | ||
| This command is most useful for testing the vector runtime | ||
| in isolation from a Zed lake. | ||
| `, | ||
| New: newCommand, | ||
| } | ||
|
|
||
| func init() { | ||
| vector.Cmd.Add(query) | ||
| } | ||
|
|
||
| type Command struct { | ||
| *root.Command | ||
| outputFlags outputflags.Flags | ||
| } | ||
|
|
||
| func newCommand(parent charm.Command, f *flag.FlagSet) (charm.Command, error) { | ||
| c := &Command{Command: parent.(*root.Command)} | ||
| c.outputFlags.SetFlags(f) | ||
| return c, nil | ||
| } | ||
|
|
||
| func (c *Command) Run(args []string) error { | ||
| ctx, cleanup, err := c.Init(&c.outputFlags) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer cleanup() | ||
| if len(args) != 2 { | ||
| return errors.New("usage: query followed by a single path argument of VNG data") | ||
| } | ||
| text := args[0] | ||
| uri, err := storage.ParseURI(args[1]) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| local := storage.NewLocalEngine() | ||
| cache := vcache.NewCache(local) | ||
| object, err := cache.Fetch(ctx, uri, ksuid.Nil) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer object.Close() | ||
| rctx := runtime.NewContext(ctx, zed.NewContext()) | ||
| puller, err := compiler.VectorCompile(rctx, text, object) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| writer, err := c.outputFlags.Open(ctx, local) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if err := zio.Copy(writer, zbuf.PullerReader(puller)); err != nil { | ||
| writer.Close() | ||
| return err | ||
| } | ||
| return writer.Close() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.