diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e9ff13a6ee..f3424f8beba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Fix `DepsStartPackageInstall` event to use package name instead of version number. ([#4482](https://github.com/dbt-labs/dbt-core/pull/4482)) - Reimplement log message to use adapter name instead of the object method. ([#4501](https://github.com/dbt-labs/dbt-core/pull/4501)) - Issue better error message for incompatible schemas ([#4470](https://github.com/dbt-labs/dbt-core/pull/4442), [#4497](https://github.com/dbt-labs/dbt-core/pull/4497)) +- Remove secrets from error related to packages. ([#4507](https://github.com/dbt-labs/dbt-core/pull/4507)) ### Docs - Fix missing data on exposures in docs ([#4467](https://github.com/dbt-labs/dbt-core/issues/4467)) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index e16da8560e4..cde9ccc77a4 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -8,7 +8,10 @@ GitProgressUpdatingExistingDependency, GitProgressPullingNewDependency, GitNothingToDo, GitProgressUpdatedCheckoutRange, GitProgressCheckedOutAt ) -import dbt.exceptions +from dbt.exceptions import ( + CommandResultError, RuntimeException, bad_package_spec, raise_git_cloning_error, + raise_git_cloning_problem +) from packaging import version @@ -22,9 +25,9 @@ def _raise_git_cloning_error(repo, revision, error): if 'usage: git' in stderr: stderr = stderr.split('\nusage: git')[0] if re.match("fatal: destination path '(.+)' already exists", stderr): - raise error + raise_git_cloning_error(error) - dbt.exceptions.bad_package_spec(repo, revision, stderr) + bad_package_spec(repo, revision, stderr) def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None): @@ -53,7 +56,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec clone_cmd.append(dirname) try: result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'}) - except dbt.exceptions.CommandResultError as exc: + except CommandResultError as exc: _raise_git_cloning_error(repo, revision, exc) if subdirectory: @@ -61,7 +64,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec clone_cmd_subdir = ['git', 'sparse-checkout', 'set', subdirectory] try: run_cmd(cwd_subdir, clone_cmd_subdir) - except dbt.exceptions.CommandResultError as exc: + except CommandResultError as exc: _raise_git_cloning_error(repo, revision, exc) if remove_git_dir: @@ -105,9 +108,9 @@ def checkout(cwd, repo, revision=None): revision = 'HEAD' try: return _checkout(cwd, repo, revision) - except dbt.exceptions.CommandResultError as exc: + except CommandResultError as exc: stderr = exc.stderr.decode('utf-8').strip() - dbt.exceptions.bad_package_spec(repo, revision, stderr) + bad_package_spec(repo, revision, stderr) def get_current_sha(cwd): @@ -131,14 +134,11 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, remove_git_dir=remove_git_dir, subdirectory=subdirectory, ) - except dbt.exceptions.CommandResultError as exc: + except CommandResultError as exc: err = exc.stderr.decode('utf-8') exists = re.match("fatal: destination path '(.+)' already exists", err) if not exists: - print( - '\nSomething went wrong while cloning {}'.format(repo) + - '\nCheck the debug logs for more information') - raise + raise_git_cloning_problem(repo) directory = None start_sha = None @@ -148,7 +148,7 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, else: matches = re.match("Cloning into '(.+)'", err.decode('utf-8')) if matches is None: - raise dbt.exceptions.RuntimeException( + raise RuntimeException( f'Error cloning {repo} - never saw "Cloning into ..." from git' ) directory = matches.group(1) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 3994fea7891..e7e9e57ef6b 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -2,8 +2,7 @@ import functools from typing import NoReturn, Optional, Mapping, Any -from dbt.logger import get_secret_env -from dbt.events.functions import fire_event +from dbt.events.functions import fire_event, scrub_secrets, env_secrets from dbt.events.types import GeneralWarningMsg, GeneralWarningException from dbt.node_types import NodeType from dbt import flags @@ -54,7 +53,7 @@ class RuntimeException(RuntimeError, Exception): def __init__(self, msg, node=None): self.stack = [] self.node = node - self.msg = msg + self.msg = scrub_secrets(msg, env_secrets()) def add_node(self, node=None): if node is not None and node is not self.node: @@ -401,8 +400,6 @@ def __init__(self, cwd, cmd, message='Error running command'): super().__init__(message) self.cwd = cwd self.cmd = cmd - for secret in get_secret_env(): - self.cmd = str(self.cmd).replace(secret, "*****") self.args = (cwd, cmd, message) def __str__(self): @@ -466,7 +463,21 @@ def raise_database_error(msg, node=None) -> NoReturn: def raise_dependency_error(msg) -> NoReturn: - raise DependencyException(msg) + raise DependencyException(scrub_secrets(msg, env_secrets())) + + +def raise_git_cloning_error(error: CommandResultError) -> NoReturn: + error.cmd = scrub_secrets(str(error.cmd), env_secrets()) + raise error + + +def raise_git_cloning_problem(repo) -> NoReturn: + repo = scrub_secrets(repo, env_secrets()) + msg = '''\ + Something went wrong while cloning {} + Check the debug logs for more information + ''' + raise RuntimeException(msg.format(repo)) def disallow_secret_env_var(env_var_name) -> NoReturn: @@ -692,9 +703,9 @@ def missing_materialization(model, adapter_type): def bad_package_spec(repo, spec, error_message): - raise InternalException( - "Error checking out spec='{}' for repo {}\n{}".format( - spec, repo, error_message)) + msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) + + raise InternalException(scrub_secrets(msg, env_secrets())) def raise_cache_inconsistent(message): @@ -999,7 +1010,7 @@ def raise_duplicate_alias( def warn_or_error(msg, node=None, log_fmt=None): if flags.WARN_ERROR: - raise_compiler_error(msg, node) + raise_compiler_error(scrub_secrets(msg, env_secrets()), node) else: fire_event(GeneralWarningMsg(msg=msg, log_fmt=log_fmt))