Cherry-pick #8769 to 6.4: The 'export config' subcommand should display field reference instead of values

CHANGELOG-developer.asciidoc

@@ -44,3 +44,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions
  `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769]

CHANGELOG.asciidoc

@@ -31,6 +31,7 @@ https:/elastic/beats/compare/v6.4.1...6.4[Check the HEAD diff]
 
 - Fix a race condition with the `add_host_metadata` and the event serialization. {pull}8223[8223] {pull}8653[8653]
 - Fix race condition when publishing monitoring data. {pull}8646[8646]
+- The export config subcommand should not display real value for field reference. {pull}8769[8769]
 
 *Auditbeat*
 

NOTICE.txt

@@ -447,8 +447,8 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-ucfg
-Version: v0.6.1
-Revision: 581f7b1fe9d84f4c18ef0694d6e0eb944a925dae
+Version: v0.6.5
+Revision: 92d43887f91851c9936621665af7f796f4d03412
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/go-ucfg/LICENSE:
 --------------------------------------------------------------------

libbeat/cmd/export/config.go

@@ -45,6 +45,8 @@ func exportConfig(settings instance.Settings, name, idxPrefix, beatVersion strin
  return fmt.Errorf("error initializing beat: %s", err)
  }
 
+ settings.DisableConfigResolver = true
+
  err = b.InitWithSettings(settings)
  if err != nil {
  return fmt.Errorf("error initializing beat: %s", err)

libbeat/cmd/instance/beat.go

@@ -61,6 +61,7 @@ import (
  "github.com/elastic/beats/libbeat/version"
  "github.com/elastic/go-sysinfo"
  "github.com/elastic/go-sysinfo/types"
+ ucfg "github.com/elastic/go-ucfg"
 
  // Register publisher pipeline modules
  _ "github.com/elastic/beats/libbeat/publisher/includes"
@@ -140,12 +141,13 @@ func init() {
 // CryptGenRandom is used.
 func initRand() {
  n, err := cryptRand.Int(cryptRand.Reader, big.NewInt(math.MaxInt64))
- seed := n.Int64()
+ var seed int64
  if err != nil {
  // fallback to current timestamp
  seed = time.Now().UnixNano()
+ } else {
+ seed = n.Int64()
  }
-
  rand.Seed(seed)
 }
 
@@ -520,8 +522,13 @@ func (b *Beat) configure(settings Settings) error {
  return fmt.Errorf("could not initialize the keystore: %v", err)
  }
 
- // TODO: Allow the options to be more flexible for dynamic changes
- common.OverwriteConfigOpts(keystore.ConfigOpts(store))
+ if settings.DisableConfigResolver {
+ common.OverwriteConfigOpts(obfuscateConfigOpts())
+ } else {
+ // TODO: Allow the options to be more flexible for dynamic changes
+ common.OverwriteConfigOpts(configOpts(store))
+ }
+
  b.keystore = store
  err = cloudid.OverwriteSettings(cfg)
  if err != nil {
@@ -828,3 +835,23 @@ func logSystemInfo(info beat.Info) {
  }
  }
 }
+
+// configOpts returns ucfg config options with a resolver linked to the current keystore.
+// TODO: Refactor to allow insert into the config option array without having to redefine everything
+func configOpts(store keystore.Keystore) []ucfg.Option {
+ return []ucfg.Option{
+ ucfg.PathSep("."),
+ ucfg.Resolve(keystore.ResolverWrap(store)),
+ ucfg.ResolveEnv,
+ ucfg.VarExp,
+ }
+}
+
+// obfuscateConfigOpts disables any resolvers in the configuration, instead we return the field
+// reference string directly.
+func obfuscateConfigOpts() []ucfg.Option {
+ return []ucfg.Option{
+ ucfg.PathSep("."),
+ ucfg.ResolveNOOP,
+ }
+}

libbeat/cmd/instance/settings.go

@@ -26,10 +26,11 @@ import (
 
 // Settings contains basic settings for any beat to pass into GenRootCmd
 type Settings struct {
- Name string
- IndexPrefix string
- Version string
- Monitoring report.Settings
- RunFlags *pflag.FlagSet
- ConfigOverrides *common.Config
+ Name string
+ IndexPrefix string
+ Version string
+ Monitoring report.Settings
+ RunFlags *pflag.FlagSet
+ ConfigOverrides *common.Config
+ DisableConfigResolver bool
 }

libbeat/keystore/keystore.go

@@ -120,14 +120,3 @@ func ResolverWrap(keystore Keystore) func(string) (string, error) {
  return string(v), nil
  }
 }
-
-// ConfigOpts returns ucfg config options with a resolver linked to the current keystore.
-// TODO: Refactor to allow insert into the config option array without having to redefine everything
-func ConfigOpts(keystore Keystore) []ucfg.Option {
- return []ucfg.Option{
- ucfg.PathSep("."),
- ucfg.Resolve(ResolverWrap(keystore)),
- ucfg.ResolveEnv,
- ucfg.VarExp,
- }
-}

libbeat/tests/system/beat/beat.py

@@ -35,29 +35,37 @@ class Proc(object):
  the object gets collected.
  """
 
- def __init__(self, args, outputfile):
+ def __init__(self, args, outputfile, env={}):
  self.args = args
  self.output = open(outputfile, "ab")
  self.stdin_read, self.stdin_write = os.pipe()
+ self.env = env
 
  def start(self):
 
  if sys.platform.startswith("win"):
+ # ensure that the environment is inherited to the subprocess.
+ variables = os.environ.copy()
+ variables = variables.update(self.env)
+
  self.proc = subprocess.Popen(
  self.args,
  stdin=self.stdin_read,
  stdout=self.output,
  stderr=subprocess.STDOUT,
  bufsize=0,
- creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+ creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+ env=variables)
  else:
  self.proc = subprocess.Popen(
  self.args,
  stdin=self.stdin_read,
  stdout=self.output,
  stderr=subprocess.STDOUT,
  bufsize=0,
- )
+ env=self.env)
+ # If a "No such file or directory" error points you here, run
+ # "make metricbeat.test" on metricbeat folder
  return self.proc
 
  def kill(self):
@@ -140,15 +148,16 @@ def run_beat(self,
  output=None,
  logging_args=["-e", "-v", "-d", "*"],
  extra_args=[],
- exit_code=None):
+ exit_code=None,
+ env={}):
  """
  Executes beat.
  Waits for the process to finish before returning to
  the caller.
  """
  proc = self.start_beat(cmd=cmd, config=config, output=output,
  logging_args=logging_args,
- extra_args=extra_args)
+ extra_args=extra_args, env=env)
  if exit_code != None:
  return proc.check_wait(exit_code)
 
@@ -159,7 +168,8 @@ def start_beat(self,
  config=None,
  output=None,
  logging_args=["-e", "-v", "-d", "*"],
- extra_args=[]):
+ extra_args=[],
+ env={}):
  """
  Starts beat and returns the process handle. The
  caller is responsible for stopping / waiting for the
@@ -190,7 +200,7 @@ def start_beat(self,
  if extra_args:
  args.extend(extra_args)
 
- proc = Proc(args, os.path.join(self.working_dir, output))
+ proc = Proc(args, os.path.join(self.working_dir, output), env)
  proc.start()
  return proc
 

libbeat/tests/system/test_cmd.py

@@ -149,6 +149,24 @@ def test_export_config(self):
  assert self.log_contains("filename: mockbeat")
  assert self.log_contains("period: 1234")
 
+ def test_export_config_environment_variable(self):
+ """
+ Test export config works but doesn"t expose environment variable.
+ """
+ self.render_config_template("mockbeat",
+ os.path.join(self.working_dir,
+ "libbeat.yml"),
+ metrics_period="${METRIC_PERIOD}")
+
+ exit_code = self.run_beat(
+ logging_args=[],
+ extra_args=["export", "config"],
+ config="libbeat.yml", env={'METRIC_PERIOD': '1234'})
+
+ assert exit_code == 0
+ assert self.log_contains("filename: mockbeat")
+ assert self.log_contains("period: ${METRIC_PERIOD}")
+
  def test_export_template(self):
  """
  Test export template works

libbeat/tests/system/test_keystore.py

@@ -70,3 +70,24 @@ def test_keystore_with_nested_key(self):
  self.wait_until(lambda: self.log_contains("no such host"))
  assert self.log_contains(secret)
  proc.check_kill_and_wait()
+
+ def test_export_config_with_keystore(self):
+ """
+ Test export config works and doesn't expose keystore value
+ """
+ key = "asecret"
+ secret = "asecretvalue"
+
+ self.render_config_template(keystore_path=self.keystore_path, elasticsearch={
+ 'hosts': "${%s}" % key
+ })
+
+ exit_code = self.run_beat(extra_args=["keystore", "create"])
+ assert exit_code == 0
+
+ self.add_secret(key, value=secret)
+ exit_code = self.run_beat(extra_args=["export", "config"])
+
+ assert exit_code == 0
+ assert self.log_contains(secret) == False
+ assert self.log_contains("${%s}" % key)

vendor/vendor.json

@@ -672,12 +672,12 @@
  "versionExact": "v0.0.3"
  },
  {
- "checksumSHA1": "MK8/w0Idj7kRBUiBabARPdm9hOo=",
+ "checksumSHA1": "Yb61Nqnh+3igFci61hv9WYgk/hc=",
  "path": "github.com/elastic/go-ucfg",
- "revision": "581f7b1fe9d84f4c18ef0694d6e0eb944a925dae",
- "revisionTime": "2018-07-13T14:04:29Z",
- "version": "v0.6.1",
- "versionExact": "v0.6.1"
+ "revision": "92d43887f91851c9936621665af7f796f4d03412",
+ "revisionTime": "2018-10-26T17:42:06Z",
+ "version": "v0.6.5",
+ "versionExact": "v0.6.5"
  },
  {
  "checksumSHA1": "X+R/CD8SokJrmlxFTx2nSevRDhQ=",