Draft: Toml (#1439)

* toml wip * wip * Fixed auto parsing toml * Added build flag not to include toml * Parse toml docs and tests * minor updates
mikefarah · Mar 25, 2023 · 7103b78 · 7103b78
1 parent 47f4ddc
commit 7103b78
Show file tree

Hide file tree

Showing 16 changed files with 828 additions and 23 deletions.
diff --git a/.gitignore b/.gitignore
@@ -41,6 +41,8 @@ yq*.snap
 
 test.yml
 test*.yml
+test*.xml
+test*.toml
 test*.yaml
 0.yml
 1.yml

diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh
@@ -2,6 +2,7 @@
 
 setUp() {
  rm test*.yml 2>/dev/null || true
+ rm test*.toml 2>/dev/null || true
  rm test*.tfstate 2>/dev/null || true
  rm test*.json 2>/dev/null || true
  rm test*.properties 2>/dev/null || true
@@ -30,6 +31,26 @@ EOM
  assertEquals "$expected" "$X"
 }
 
+testInputToml() {
+ cat >test.toml <<EOL
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+EOL
+
+ read -r -d '' expected << EOM
+owner:
+ name: Tom Preston-Werner
+ dob: 1979-05-27T07:32:00-08:00
+EOM
+
+ X=$(./yq -oy test.toml)
+ assertEquals "$expected" "$X"
+
+ X=$(./yq ea -oy test.toml)
+ assertEquals "$expected" "$X"
+}
+
 testInputTfstate() {
  cat >test.tfstate <<EOL
 { "mike" : { "things": "cool" } }

diff --git a/cmd/root.go b/cmd/root.go
@@ -65,8 +65,8 @@ yq -P sample.json
  panic(err)
  }
 
- rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
- rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+ rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
+ rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
 
  rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
  rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

diff --git a/cmd/utils.go b/cmd/utils.go
@@ -75,6 +75,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
  }
  } else if isAutomaticOutputFormat() {
  // automatic input worked, we can do it for output too unless specified
+ if inputFormat == "toml" {
+ return "", nil, fmt.Errorf("toml is not yet supported as an output format. Please specify another output format using the [--output-format/-o] flag")
+ }
  outputFormat = inputFormat
  }
  } else if isAutomaticOutputFormat() {
@@ -137,6 +140,8 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
  return yqlib.NewCSVObjectDecoder(','), nil
  case yqlib.TSVObjectInputFormat:
  return yqlib.NewCSVObjectDecoder('\t'), nil
+ case yqlib.TomlInputFormat:
+ return yqlib.NewTomlDecoder(), nil
  case yqlib.YamlInputFormat:
  prefs := yqlib.ConfiguredYamlPreferences
  prefs.EvaluateTogether = evaluateTogether

diff --git a/examples/sample.toml b/examples/sample.toml
@@ -0,0 +1,26 @@
+
+
+# This is a TOML document
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+
+[database]
+enabled = true
+ports = [ 8000, 8001, 8002 ]
+data = [ ["delta", "phi"], [3.14] ]
+temp_targets = { cpu = 79.5, case = 72.0 }
+
+[servers]
+
+[servers.alpha]
+ip = "10.0.0.1"
+role = "frontend"
+
+[servers.beta]
+ip = "10.0.0.2"
+role = "backend"
+
diff --git a/go.mod b/go.mod
@@ -11,6 +11,7 @@ require (
  github.com/goccy/go-yaml v1.10.0
  github.com/jinzhu/copier v0.3.5
  github.com/magiconair/properties v1.8.7
+ github.com/pelletier/go-toml/v2 v2.0.6
  github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
  github.com/spf13/cobra v1.6.1
  github.com/spf13/pflag v1.0.5

diff --git a/go.sum b/go.sum
@@ -6,8 +6,9 @@ github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0q
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
@@ -36,6 +37,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
+github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -49,9 +52,14 @@ github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

diff --git a/pkg/yqlib/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go
@@ -4,13 +4,16 @@ import (
  "fmt"
 
  logging "gopkg.in/op/go-logging.v1"
+ yaml "gopkg.in/yaml.v3"
 )
 
 type DataTreeNavigator interface {
  // given the context and a expressionNode,
  // this will process the against the given expressionNode and return
  // a new context of matching candidates
  GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
+
+ DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
 }
 
 type dataTreeNavigator struct {
@@ -20,6 +23,27 @@ func NewDataTreeNavigator() DataTreeNavigator {
  return &dataTreeNavigator{}
 }
 
+func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
+
+ rhsCandidateNode := &CandidateNode{
+ Path: path,
+ Node: rhsNode,
+ }
+
+ assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+ rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+
+ assignmentOpNode := &ExpressionNode{
+ Operation: assignmentOp,
+ LHS: createTraversalTree(path, traversePreferences{}, false),
+ RHS: &ExpressionNode{Operation: rhsOp},
+ }
+
+ _, err := d.GetMatchingNodes(context, assignmentOpNode)
+ return err
+}
+
 func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
  if expressionNode == nil {
  log.Debugf("getMatchingNodes - nothing to do")

diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
@@ -16,6 +16,7 @@ const (
  JsonInputFormat
  CSVObjectInputFormat
  TSVObjectInputFormat
+ TomlInputFormat
  UriInputFormat
 )
 
@@ -38,8 +39,10 @@ func InputFormatFromString(format string) (InputFormat, error) {
  return CSVObjectInputFormat, nil
  case "tsv", "t":
  return TSVObjectInputFormat, nil
+ case "toml":
+ return TomlInputFormat, nil
  default:
- return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
+ return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
  }
 }
 

diff --git a/pkg/yqlib/decoder_properties.go b/pkg/yqlib/decoder_properties.go
@@ -95,23 +95,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
 
  rhsNode.Tag = guessTagFromCustomType(rhsNode)
 
- rhsCandidateNode := &CandidateNode{
- Path: path,
- Node: rhsNode,
- }
-
- assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
- rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
-
- assignmentOpNode := &ExpressionNode{
- Operation: assignmentOp,
- LHS: createTraversalTree(path, traversePreferences{}, false),
- RHS: &ExpressionNode{Operation: rhsOp},
- }
-
- _, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
- return err
+ return dec.d.DeeplyAssign(context, path, rhsNode)
 }
 
 func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {

diff --git a/pkg/yqlib/decoder_test.go b/pkg/yqlib/decoder_test.go
@@ -3,6 +3,7 @@ package yqlib
 import (
  "bufio"
  "bytes"
+ "fmt"
  "strings"
 )
 
@@ -62,7 +63,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
 
  result, err := processFormatScenario(s, decoder, encoder)
  if err != nil {
- panic(err)
+ panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
  }
  return result