From 8b31d894aa0e296d2ae0c384594e1dcaf56894e1 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 11:41:15 -0500 Subject: [PATCH 01/18] add struct logging to base.py --- core/dbt/events/types.py | 72 +++++++++++++++++++++++++++++++++++----- core/dbt/task/base.py | 35 ++++++------------- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 222dccd5142..d9a60a05e58 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2,6 +2,9 @@ from dataclasses import dataclass from typing import Any, List, Optional, Dict from dbt.ui import warning_tag +import dataclasses +from typing import Any, List, Optional +from dbt import ui # types to represent log levels @@ -305,6 +308,28 @@ def cli_msg(self) -> str: return self.msg +@dataclass +class NewConnectionOpening(DebugLevel, CliEventABC): + connection_state: str + + def cli_msg(self) -> str: + return f"Opening a new connection, currently in state {self.connection_state}" + + +class TimingInfoCollected(DebugLevel, CliEventABC): + def cli_msg(self) -> str: + return "finished collecting timing info" + + +@dataclass +class MergedFromState(DebugLevel, CliEventABC): + nbr_merged: int + sample: List + + def cli_msg(self) -> str: + return f"Merged {self.nbr_merged} items from state (sample: {self.sample})" + + @dataclass class MissingProfileTarget(InfoLevel, CliEventABC): profile_name: str @@ -336,25 +361,52 @@ def cli_msg(self) -> str: @dataclass -class NewConnectionOpening(DebugLevel, CliEventABC): - connection_state: str +class CatchRunException(ShowException, DebugLevel, CliEventABC): + build_path: Any = '' + exc: Exception = '' def cli_msg(self) -> str: - return f"Opening a new connection, currently in state {self.connection_state}" + INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If the \ + error persists, open an issue at https://github.com/dbt-labs/dbt-core + """.strip() + prefix = f'Internal error executing {self.build_path}' + error = "{prefix}\n{error}\n\n{note}".format( + prefix=ui.red(prefix), + error=str(self.exc).strip(), + note=INTERNAL_ERROR_STRING + ) + return str(self.exc) -class TimingInfoCollected(DebugLevel, CliEventABC): +@dataclass +class HandleInternalException(ShowException, DebugLevel, CliEventABC): + exc: Exception = '' + def cli_msg(self) -> str: - return "finished collecting timing info" + return str(self.exc) @dataclass -class MergedFromState(DebugLevel, CliEventABC): - nbr_merged: int - sample: List +class MessageHandleGenericException(ErrorLevel, CliEventABC): + build_path: str + unique_id: str + exc: Exception def cli_msg(self) -> str: - return f"Merged {self.nbr_merged} items from state (sample: {self.sample})" + node_description = self.build_path + if node_description is None: + node_description = self.unique_id + prefix = "Unhandled error while executing {}".format(node_description) + return "{prefix}\n{error}".format( + prefix=ui.red(prefix), + error=str(self.exc).strip() + ) + + +@dataclass +class DetailsHandleGenericException(ShowException, DebugLevel, CliEventABC): + def cli_msg(self) -> str: + return '' @dataclass @@ -725,3 +777,5 @@ def cli_msg(self) -> str: PartialParsingDeletedExposure(unique_id='') InvalidDisabledSourceInTestNode(msg='') InvalidRefInTestNode(msg='') + MessageHandleGenericException(build_path='', unique_id='', exc=Exception('')) + DetailsHandleGenericException() diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index af0eb3de818..567aa2c4177 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -17,6 +17,11 @@ InternalException ) from dbt.logger import GLOBAL_LOGGER as logger, log_manager +from dbt.events.functions import fire_event +from dbt.events.types import( + CatchRunException, HandleInternalException, MessageHandleGenericException, + DetailsHandleGenericException, +) from .printer import print_skip_caused_by_error, print_skip_line @@ -165,11 +170,6 @@ def from_args(cls, args): return super().from_args(args) -INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If \ -the error persists, open an issue at https://github.com/dbt-labs/dbt-core -""".strip() - - class ExecutionContext: """During execution and error handling, dbt makes use of mutable state: timing information and the newest (compiled vs executed) form of the node. @@ -307,33 +307,20 @@ def _handle_catchable_exception(self, e, ctx): if e.node is None: e.add_node(ctx.node) - logger.debug(str(e), exc_info=True) + fire_event(CatchRunException(exc=e)) return str(e) def _handle_internal_exception(self, e, ctx): build_path = self.node.build_path - prefix = 'Internal error executing {}'.format(build_path) - error = "{prefix}\n{error}\n\n{note}".format( - prefix=ui.red(prefix), - error=str(e).strip(), - note=INTERNAL_ERROR_STRING - ) - logger.debug(error, exc_info=True) + fire_event(HandleInternalException(build_path=build_path, exc=e)) return str(e) def _handle_generic_exception(self, e, ctx): - node_description = self.node.build_path - if node_description is None: - node_description = self.node.unique_id - prefix = "Unhandled error while executing {}".format(node_description) - error = "{prefix}\n{error}".format( - prefix=ui.red(prefix), - error=str(e).strip() - ) - - logger.error(error) - logger.debug('', exc_info=True) + fire_event(MessageHandleGenericException(build_path=self.node.build_path, + unique_id=self.node.unique_id, + exc=e)) + fire_event(DetailsHandleGenericException()) return str(e) def handle_exception(self, e, ctx): From 90385c7afb8e5445db3fc44562c650ab2584034a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 11:41:34 -0500 Subject: [PATCH 02/18] struct logging in run_operation --- core/dbt/events/types.py | 15 +++++++++++++++ core/dbt/task/run_operation.py | 17 ++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index d9a60a05e58..392ed5ccb71 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -697,6 +697,19 @@ class InvalidRefInTestNode(WarnLevel, CliEventABC): def cli_msg(self) -> str: return warning_tag(self.msg) +class RunningOperationCaughtError(ShowException, ErrorLevel, CliEventABC): + exc: Exception + + def cli_msg(self) -> str: + return f'Encountered an error while running operation: {self.exc}' + + +@dataclass +class RunningOperationUncaughtError(ShowException, ErrorLevel, CliEventABC): + exc: Exception + + def cli_msg(self) -> str: + return f'Encountered an error while running operation: {self.exc}' # since mypy doesn't run on every file we need to suggest to mypy that every @@ -779,3 +792,5 @@ def cli_msg(self) -> str: InvalidRefInTestNode(msg='') MessageHandleGenericException(build_path='', unique_id='', exc=Exception('')) DetailsHandleGenericException() + RunningOperationCaughtError(exc=Exception('')) + RunningOperationUncaughtError(exc=Exception('')) diff --git a/core/dbt/task/run_operation.py b/core/dbt/task/run_operation.py index 23bff66c5b0..35bddfdff63 100644 --- a/core/dbt/task/run_operation.py +++ b/core/dbt/task/run_operation.py @@ -10,7 +10,10 @@ from dbt.config.utils import parse_cli_vars from dbt.contracts.results import RunOperationResultsArtifact from dbt.exceptions import InternalException -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.events.functions import fire_event +from dbt.events.types import ( + RunningOperationCaughtError, RunningOperationUncaughtError +) class RunOperationTask(ManifestTask): @@ -53,18 +56,10 @@ def run(self) -> RunOperationResultsArtifact: try: self._run_unsafe() except dbt.exceptions.Exception as exc: - logger.error( - 'Encountered an error while running operation: {}' - .format(exc) - ) - logger.debug('', exc_info=True) + fire_event(RunningOperationCaughtError(exc=exc)) success = False except Exception as exc: - logger.error( - 'Encountered an uncaught exception while running operation: {}' - .format(exc) - ) - logger.debug('', exc_info=True) + fire_event(RunningOperationUncaughtError(exc=exc)) success = False else: success = True From a75d647aae1a0088095b9e1933c2ab8d11484231 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 1 Nov 2021 17:29:35 -0500 Subject: [PATCH 03/18] add struct logging to base --- core/dbt/events/types.py | 120 +++++++++++++++++++++++++++++++++++++++ core/dbt/task/base.py | 53 +++++++---------- 2 files changed, 141 insertions(+), 32 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 392ed5ccb71..4e3023f92e6 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -712,6 +712,114 @@ def cli_msg(self) -> str: return f'Encountered an error while running operation: {self.exc}' +class DbtProjectError(ErrorLevel, CliEventABC): + def cli_msg(self) -> str: + return "Encountered an error while reading the project:" + + +@dataclass +class DbtProjectErrorException(ErrorLevel, CliEventABC): + exc: Exception + + def cli_msg(self) -> str: + return f" ERROR: {str(self.exc)}" + + +class DbtProfileError(ErrorLevel, CliEventABC): + def cli_msg(self) -> str: + return "Encountered an error while reading profiles:" + + +@dataclass +class DbtProfileErrorException(ErrorLevel, CliEventABC): + exc: Exception + + def cli_msg(self) -> str: + return f" ERROR: {str(self.exc)}" + + +class ProfileListTitle(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "Defined profiles:" + + +@dataclass +class ListSingleProfile(InfoLevel, CliEventABC): + profile: str + + def cli_msg(self) -> str: + return f" - {self.profile}" + + +class NoDefinedProfiles(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "There are no profiles defined in your profiles.yml file" + + +class ProfileHelpMessage(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + PROFILES_HELP_MESSAGE = """ +For more information on configuring profiles, please consult the dbt docs: + +https://docs.getdbt.com/docs/configure-your-profile +""" + return PROFILES_HELP_MESSAGE + + +@dataclass +class CatchableExceptionOnRun(ShowException, DebugLevel, CliEventABC): + exc: Exception + + def cli_msg(self) -> str: + return str(self.exc) + + +@dataclass +class InternalExceptionOnRun(DebugLevel, CliEventABC): + build_path: str + exc: Exception + + def cli_msg(self) -> str: + prefix = 'Internal error executing {}'.format(self.build_path) + + INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If \ +the error persists, open an issue at https://github.com/dbt-labs/dbt-core +""".strip() + + return "{prefix}\n{error}\n\n{note}".format( + prefix=red(prefix), + error=str(self.exc).strip(), + note=INTERNAL_ERROR_STRING + ) + + +@dataclass +class GenericExceptionOnRun(ShowException, ErrorLevel, CliEventABC): + build_path: str + unique_id: str + exc: Exception + + def cli_msg(self) -> str: + node_description = self.build_path + if node_description is None: + node_description = self.unique_id + prefix = "Unhandled error while executing {}".format(node_description) + return "{prefix}\n{error}".format( + prefix=red(prefix), + error=str(self.exc).strip() + ) + + +@dataclass +class NodeConnectionReleaseError(ShowException, DebugLevel, CliEventABC): + node_name: str + exc: Exception + + def cli_msg(self) -> str: + return ('Error releasing connection for node {}: {!s}' + .format(self.node_name, self.exc)) + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -794,3 +902,15 @@ def cli_msg(self) -> str: DetailsHandleGenericException() RunningOperationCaughtError(exc=Exception('')) RunningOperationUncaughtError(exc=Exception('')) + DbtProjectError() + DbtProjectErrorException(exc=Exception('')) + DbtProfileError() + DbtProfileErrorException(exc=Exception('')) + ProfileListTitle() + ListSingleProfile(profile='') + NoDefinedProfiles() + ProfileHelpMessage() + CatchableExceptionOnRun(exc=Exception('')) + InternalExceptionOnRun(build_path='', exc=Exception('')) + GenericExceptionOnRun(build_path='', unique_id='', exc=Exception('')) + NodeConnectionReleaseError(node_name='', exc=Exception('')) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 567aa2c4177..abfd8f952e5 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -16,11 +16,13 @@ NotImplementedException, CompilationException, RuntimeException, InternalException ) -from dbt.logger import GLOBAL_LOGGER as logger, log_manager +from dbt.logger import GLOBAL_LOGGER as log_manager from dbt.events.functions import fire_event -from dbt.events.types import( - CatchRunException, HandleInternalException, MessageHandleGenericException, - DetailsHandleGenericException, +from dbt.events.types import ( + DbtProjectError, DbtProjectErrorException, DbtProfileError, DbtProfileErrorException, + ProfileListTitle, ListSingleProfile, NoDefinedProfiles, ProfileHelpMessage, + CatchableExceptionOnRun, InternalExceptionOnRun, GenericExceptionOnRun, + NodeConnectionReleaseError, ) from .printer import print_skip_caused_by_error, print_skip_line @@ -52,13 +54,6 @@ def read_profiles(profiles_dir=None): return profiles -PROFILES_HELP_MESSAGE = """ -For more information on configuring profiles, please consult the dbt docs: - -https://docs.getdbt.com/docs/configure-your-profile -""" - - class BaseTask(metaclass=ABCMeta): ConfigType: Union[Type[NoneConfig], Type[Project]] = NoneConfig @@ -87,28 +82,27 @@ def from_args(cls, args): try: config = cls.ConfigType.from_args(args) except dbt.exceptions.DbtProjectError as exc: - logger.error("Encountered an error while reading the project:") - logger.error(" ERROR: {}".format(str(exc))) + fire_event(DbtProjectError()) + fire_event(DbtProjectErrorException(exc=exc)) tracking.track_invalid_invocation( args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException('Could not run dbt') from exc except dbt.exceptions.DbtProfileError as exc: - logger.error("Encountered an error while reading profiles:") - logger.error(" ERROR {}".format(str(exc))) + fire_event(DbtProfileError()) + fire_event(DbtProfileErrorException(exc=exc)) all_profiles = read_profiles(flags.PROFILES_DIR).keys() if len(all_profiles) > 0: - logger.info("Defined profiles:") + fire_event(ProfileListTitle()) for profile in all_profiles: - logger.info(" - {}".format(profile)) + fire_event(ListSingleProfile(profile=profile)) else: - logger.info("There are no profiles defined in your " - "profiles.yml file") + fire_event(NoDefinedProfiles()) - logger.info(PROFILES_HELP_MESSAGE) + fire_event(ProfileHelpMessage()) tracking.track_invalid_invocation( args=args, @@ -307,20 +301,18 @@ def _handle_catchable_exception(self, e, ctx): if e.node is None: e.add_node(ctx.node) - fire_event(CatchRunException(exc=e)) + fire_event(CatchableExceptionOnRun(exc=e)) return str(e) def _handle_internal_exception(self, e, ctx): - build_path = self.node.build_path - - fire_event(HandleInternalException(build_path=build_path, exc=e)) + fire_event(InternalExceptionOnRun(build_path=self.node.build_path, exc=e)) return str(e) def _handle_generic_exception(self, e, ctx): - fire_event(MessageHandleGenericException(build_path=self.node.build_path, - unique_id=self.node.unique_id, - exc=e)) - fire_event(DetailsHandleGenericException()) + fire_event(GenericExceptionOnRun(build_path=self.node.build_path, + unique_id=self.node.unique_id, + exc=e)) + return str(e) def handle_exception(self, e, ctx): @@ -370,10 +362,7 @@ def _safe_release_connection(self): try: self.adapter.release_connection() except Exception as exc: - logger.debug( - 'Error releasing connection for node {}: {!s}\n{}' - .format(self.node.name, exc, traceback.format_exc()) - ) + fire_event(NodeConnectionReleaseError(node_name=self.node.name, exc=exc)) return str(exc) return None From ce68a04df552b85bb76001706d8a97fe9521f8e4 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 07:57:13 -0500 Subject: [PATCH 04/18] add struct logging to clean --- core/dbt/events/types.py | 33 +++++++++++++++++++++++++++++++++ core/dbt/task/clean.py | 15 +++++++++------ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 4e3023f92e6..15362d64a36 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -820,6 +820,35 @@ def cli_msg(self) -> str: .format(self.node_name, self.exc)) +@dataclass +class CheckCleanPath(InfoLevel, CliEventABC): + path: str + + def cli_msg(self) -> str: + return f"Checking {self.path}/*" + + +@dataclass +class ConfirmCleanPath(InfoLevel, CliEventABC): + path: str + + def cli_msg(self) -> str: + return f"Cleaned {self.path}/*" + + +@dataclass +class ProtectedCleanPath(InfoLevel, CliEventABC): + path: str + + def cli_msg(self) -> str: + return f"ERROR: not cleaning {self.path}/* because it is protected" + + +class FinishedCleanPaths(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "Finished cleaning all paths." + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -914,3 +943,7 @@ def cli_msg(self) -> str: InternalExceptionOnRun(build_path='', exc=Exception('')) GenericExceptionOnRun(build_path='', unique_id='', exc=Exception('')) NodeConnectionReleaseError(node_name='', exc=Exception('')) + CheckCleanPath(path='') + ConfirmCleanPath(path='') + ProtectedCleanPath(path='') + FinishedCleanPaths() diff --git a/core/dbt/task/clean.py b/core/dbt/task/clean.py index 218d8cb4eba..1d84fd0db7f 100644 --- a/core/dbt/task/clean.py +++ b/core/dbt/task/clean.py @@ -4,7 +4,10 @@ from dbt import deprecations from dbt.task.base import BaseTask, move_to_nearest_project_dir -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.events.functions import fire_event +from dbt.events.types import ( + CheckCleanPath, ConfirmCleanPath, ProtectedCleanPath, FinishedCleanPaths +) from dbt.config import UnsetProfileConfig @@ -38,11 +41,11 @@ def run(self): self.config.packages_install_path != 'dbt_modules'): deprecations.warn('install-packages-path') for path in self.config.clean_targets: - logger.info("Checking {}/*".format(path)) + fire_event(CheckCleanPath(path=path)) if not self.__is_protected_path(path): shutil.rmtree(path, True) - logger.info(" Cleaned {}/*".format(path)) + fire_event(ConfirmCleanPath(path=path)) else: - logger.info("ERROR: not cleaning {}/* because it is " - "protected".format(path)) - logger.info("Finished cleaning all paths.") + fire_event(ProtectedCleanPath(path=path)) + + fire_event(FinishedCleanPaths()) From b5f674684bc688a52905127690e9d93112747f61 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 08:48:06 -0500 Subject: [PATCH 05/18] add struct logging to debug --- core/dbt/events/types.py | 18 ++++++++++++++++++ core/dbt/task/debug.py | 5 +++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 15362d64a36..951d149ba28 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -849,6 +849,23 @@ def cli_msg(self) -> str: return "Finished cleaning all paths." +@dataclass +class OpenCommand(InfoLevel, CliEventABC): + open_cmd: str + profiles_dir: str + + def cli_msg(self) -> str: + PROFILE_DIR_MESSAGE = """To view your profiles.yml file, run: + +{open_cmd} {profiles_dir}""" + message = PROFILE_DIR_MESSAGE.format( + open_cmd=self.open_cmd, + profiles_dir=self.profiles_dir + ) + + return message + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -947,3 +964,4 @@ def cli_msg(self) -> str: ConfirmCleanPath(path='') ProtectedCleanPath(path='') FinishedCleanPaths() + OpenCommand(open_cmd='', profiles_dir='') diff --git a/core/dbt/task/debug.py b/core/dbt/task/debug.py index c26c2ad0f39..2b5011b7c15 100644 --- a/core/dbt/task/debug.py +++ b/core/dbt/task/debug.py @@ -4,7 +4,8 @@ import sys from typing import Optional, Dict, Any, List -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.events.functions import fire_event +from dbt.events.types import OpenCommand from dbt import flags import dbt.clients.system import dbt.exceptions @@ -108,7 +109,7 @@ def path_info(self): profiles_dir=self.profiles_dir ) - logger.info(message) + fire_event(OpenCommand(open_cmd=open_cmd, profiles_dir=self.profiles_dir)) def run(self): if self.args.config_dir: From 069427667391602f5743e7cb4fb7d7388c2e6421 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 08:49:20 -0500 Subject: [PATCH 06/18] add struct logging to deps --- core/dbt/events/types.py | 63 ++++++++++++++++++++++++++++++++++++++++ core/dbt/task/deps.py | 25 ++++++++-------- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 951d149ba28..f0bf5ffbbdc 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -866,6 +866,62 @@ def cli_msg(self) -> str: return message +class DepsNoPackagesFound(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return 'Warning: No packages were found in packages.yml' + + +@dataclass +class DepsStartPackageInstall(InfoLevel, CliEventABC): + package: str + + def cli_msg(self) -> str: + return f"Installing {self.package}" + + +@dataclass +class DepsInstallInfo(InfoLevel, CliEventABC): + version_name: str + + def cli_msg(self) -> str: + return f" Installed from {self.version_name}" + + +@dataclass +class DepsUpdateAvailable(InfoLevel, CliEventABC): + version_latest: str + + def cli_msg(self) -> str: + return f" Updated version available: {self.version_latest}" + + +@dataclass +class DepsUTD(InfoLevel, CliEventABC): + version_latest: str + + def cli_msg(self) -> str: + return " Up to date!" + + +@dataclass +class DepsListSubdirectory(InfoLevel, CliEventABC): + subdirectory: str + + def cli_msg(self) -> str: + return f" and subdirectory {self.subdirectory}" + + +@dataclass +class DepsNotifyUpdatesAvailable(InfoLevel, CliEventABC): + packages: List + + def cli_msg(self) -> str: + return ('\nUpdates available for packages: {} \ + \nUpdate your versions in packages.yml, then run dbt deps', + self.packages) + + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -965,3 +1021,10 @@ def cli_msg(self) -> str: ProtectedCleanPath(path='') FinishedCleanPaths() OpenCommand(open_cmd='', profiles_dir='') + DepsNoPackagesFound() + DepsStartPackageInstall(package='') + DepsInstallInfo(version_name='') + DepsUpdateAvailable(version_latest='') + DepsListSubdirectory(subdirectory='') + DepsNotifyUpdatesAvailable(packages=[]) + diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 573098a8080..081bef616b8 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -8,7 +8,11 @@ from dbt.deps.base import downloads_directory from dbt.deps.resolver import resolve_packages -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.events.functions import fire_event +from dbt.events.types import ( + DepsNoPackagesFound, DepsStartPackageInstall, DepsUpdateAvailable, DepsUTD, + DepsInstallInfo, DepsListSubdirectory, DepsNotifyUpdatesAvailable +) from dbt.clients import system from dbt.task.base import BaseTask, move_to_nearest_project_dir @@ -46,7 +50,7 @@ def run(self): system.make_directory(self.config.packages_install_path) packages = self.config.packages.packages if not packages: - logger.info('Warning: No packages were found in packages.yml') + fire_event(DepsNoPackagesFound()) return with downloads_directory(): @@ -62,30 +66,25 @@ def run(self): source_type = package.source_type() version = package.get_version() - logger.info('Installing {}', package) + fire_event(DepsStartPackageInstall(package=package)) package.install(self.config, renderer) - logger.info(' Installed from {}', - package.nice_version_name()) + fire_event(DepsInstallInfo(version_name=package.nice_version_name())) if source_type == 'hub': version_latest = package.get_version_latest() if version_latest != version: packages_to_upgrade.append(package_name) - logger.info(' Updated version available: {}', - version_latest) + fire_event(DepsUpdateAvailable(version_latest=version_latest)) else: - logger.info(' Up to date!') + fire_event(DepsUTD()) if package.get_subdirectory(): - logger.info(' and subdirectory {}', - package.get_subdirectory()) + fire_event(DepsListSubdirectory(subdirectory=package.get_subdirectory())) self.track_package_install( package_name=package_name, source_type=source_type, version=version) if packages_to_upgrade: - logger.info('\nUpdates available for packages: {} \ - \nUpdate your versions in packages.yml, then run dbt deps', - packages_to_upgrade) + fire_event(DepsNotifyUpdatesAvailable(packages=packages_to_upgrade)) @classmethod def from_args(cls, args): From 7973a09b6420e386e3ddaf5b619e2b9e145c1f3a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 08:54:24 -0500 Subject: [PATCH 07/18] fix errors --- core/dbt/events/types.py | 14 ++++++-------- core/dbt/task/base.py | 2 -- core/dbt/task/clean.py | 2 +- core/dbt/task/debug.py | 6 ------ 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index f0bf5ffbbdc..2157eec23a5 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -374,8 +374,8 @@ def cli_msg(self) -> str: prefix=ui.red(prefix), error=str(self.exc).strip(), note=INTERNAL_ERROR_STRING - ) - return str(self.exc) + ) + return error @dataclass @@ -777,7 +777,7 @@ def cli_msg(self) -> str: @dataclass class InternalExceptionOnRun(DebugLevel, CliEventABC): build_path: str - exc: Exception + exc: Exception def cli_msg(self) -> str: prefix = 'Internal error executing {}'.format(self.build_path) @@ -787,7 +787,7 @@ def cli_msg(self) -> str: """.strip() return "{prefix}\n{error}\n\n{note}".format( - prefix=red(prefix), + prefix=ui.red(prefix), error=str(self.exc).strip(), note=INTERNAL_ERROR_STRING ) @@ -805,7 +805,7 @@ def cli_msg(self) -> str: node_description = self.unique_id prefix = "Unhandled error while executing {}".format(node_description) return "{prefix}\n{error}".format( - prefix=red(prefix), + prefix=ui.red(prefix), error=str(self.exc).strip() ) @@ -858,7 +858,7 @@ def cli_msg(self) -> str: PROFILE_DIR_MESSAGE = """To view your profiles.yml file, run: {open_cmd} {profiles_dir}""" - message = PROFILE_DIR_MESSAGE.format( + message = PROFILE_DIR_MESSAGE.format( open_cmd=self.open_cmd, profiles_dir=self.profiles_dir ) @@ -921,7 +921,6 @@ def cli_msg(self) -> str: self.packages) - # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -1027,4 +1026,3 @@ def cli_msg(self) -> str: DepsUpdateAvailable(version_latest='') DepsListSubdirectory(subdirectory='') DepsNotifyUpdatesAvailable(packages=[]) - diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index abfd8f952e5..94b183c0dd3 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -1,12 +1,10 @@ import os import threading import time -import traceback from abc import ABCMeta, abstractmethod from typing import Type, Union, Dict, Any, Optional from dbt import tracking -from dbt import ui from dbt import flags from dbt.contracts.graph.manifest import Manifest from dbt.contracts.results import ( diff --git a/core/dbt/task/clean.py b/core/dbt/task/clean.py index 1d84fd0db7f..6dfc8b3328b 100644 --- a/core/dbt/task/clean.py +++ b/core/dbt/task/clean.py @@ -47,5 +47,5 @@ def run(self): fire_event(ConfirmCleanPath(path=path)) else: fire_event(ProtectedCleanPath(path=path)) - + fire_event(FinishedCleanPaths()) diff --git a/core/dbt/task/debug.py b/core/dbt/task/debug.py index 2b5011b7c15..983a76acdfd 100644 --- a/core/dbt/task/debug.py +++ b/core/dbt/task/debug.py @@ -103,12 +103,6 @@ def project_profile(self): def path_info(self): open_cmd = dbt.clients.system.open_dir_cmd() - - message = PROFILE_DIR_MESSAGE.format( - open_cmd=open_cmd, - profiles_dir=self.profiles_dir - ) - fire_event(OpenCommand(open_cmd=open_cmd, profiles_dir=self.profiles_dir)) def run(self): From fb1c5594972c168d31bb068533dd4d26c45ac940 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 09:22:02 -0500 Subject: [PATCH 08/18] add struct logging to run.py --- core/dbt/events/types.py | 35 +++++++++++++++++++++++++++++++++++ core/dbt/task/run.py | 22 +++++++++------------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 2157eec23a5..29a8bc45b78 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -921,6 +921,37 @@ def cli_msg(self) -> str: self.packages) +@dataclass +class DatabaseErrorRunning(InfoLevel, CliEventABC): + hook_type: str + + def cli_msg(self) -> str: + return f"Database error while running {self.hook_type}" + + +class EmptyLine(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return '' + + +@dataclass +class HooksRunning(InfoLevel, CliEventABC): + num_hooks: int + hook_type: str + + def cli_msg(self) -> str: + plural = 'hook' if self.num_hooks == 1 else 'hooks' + return f"Running {self.num_hooks} {self.hook_type} {plural}" + + +@dataclass +class HookFinished(InfoLevel, CliEventABC): + stat_line: str + execution: str + + def cli_msg(self) -> str: + return f"Finished running {self.stat_line}{self.execution}." + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -1026,3 +1057,7 @@ def cli_msg(self) -> str: DepsUpdateAvailable(version_latest='') DepsListSubdirectory(subdirectory='') DepsNotifyUpdatesAvailable(packages=[]) + DatabaseErrorRunning(hook_type='') + EmptyLine() + HooksRunning(num_hooks=0, hook_type='') + HookFinished(stat_line='', execution='') diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index bb3223fc39d..f42425255c6 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -32,14 +32,14 @@ RuntimeException, missing_materialization, ) +from dbt.events.functions import fire_event +from dbt.events.types import DatabaseErrorRunning, EmptyLine, HooksRunning, HookFinished from dbt.logger import ( - GLOBAL_LOGGER as logger, TextOnly, HookMetadata, UniqueID, TimestampNamed, DbtModelState, - print_timestamped_line, ) from dbt.graph import ResourceTypeSelector from dbt.hooks import get_hook_dict @@ -308,12 +308,10 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): return num_hooks = len(ordered_hooks) - plural = 'hook' if num_hooks == 1 else 'hooks' with TextOnly(): - print_timestamped_line("") - print_timestamped_line( - 'Running {} {} {}'.format(num_hooks, hook_type, plural) - ) + fire_event(EmptyLine()) + fire_event(HooksRunning(num_hooks=num_hooks, hook_type=hook_type)) + startctx = TimestampNamed('node_started_at') finishctx = TimestampNamed('node_finished_at') @@ -344,7 +342,7 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): self._total_executed += len(ordered_hooks) with TextOnly(): - print_timestamped_line("") + fire_event(EmptyLine()) def safe_run_hooks( self, adapter, hook_type: RunHookType, extra_context: Dict[str, Any] @@ -352,7 +350,7 @@ def safe_run_hooks( try: self.run_hooks(adapter, hook_type, extra_context) except RuntimeException: - logger.info("Database error while running {}".format(hook_type)) + fire_event(DatabaseErrorRunning(hook_type)) raise def print_results_line(self, results, execution_time): @@ -366,10 +364,8 @@ def print_results_line(self, results, execution_time): execution_time=execution_time) with TextOnly(): - print_timestamped_line("") - print_timestamped_line( - "Finished running {stat_line}{execution}." - .format(stat_line=stat_line, execution=execution)) + fire_event(EmptyLine()) + fire_event(HookFinished(stat_line=stat_line, execution=execution)) def _get_deferred_manifest(self) -> Optional[WritableManifest]: if not self.args.defer: From 517a5c252182dce6930d5da2136eca47017e51a3 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 09:22:51 -0500 Subject: [PATCH 09/18] fixed flake error --- core/dbt/events/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 29a8bc45b78..91e96fbfd5e 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -952,6 +952,7 @@ class HookFinished(InfoLevel, CliEventABC): def cli_msg(self) -> str: return f"Finished running {self.stat_line}{self.execution}." + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so From f95c5d82bf60ece76d271e0b8ded03c58d405325 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 09:49:51 -0500 Subject: [PATCH 10/18] add struct logging to geneerate --- core/dbt/events/types.py | 32 ++++++++++++++++++++++++++++++++ core/dbt/task/generate.py | 22 ++++++++-------------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 91e96fbfd5e..f875acb97a8 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -953,6 +953,33 @@ def cli_msg(self) -> str: return f"Finished running {self.stat_line}{self.execution}." +@dataclass +class WriteCatalogFailure(ErrorLevel, CliEventABC): + num_exceptions: int + + def cli_msg(self) -> str: + return (f"dbt encountered {self.num_exceptions} failure{(self.num_exceptions != 1) * 's'} " + "while writing the catalog") + + +@dataclass +class CatalogWritten(InfoLevel, CliEventABC): + path: str + + def cli_msg(self) -> str: + return f"Catalog written to {self.path}" + + +class CannotGenerateDocs(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "compile failed, cannot generate docs" + + +class BuildingCatalog(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "Building catalog" + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -1062,3 +1089,8 @@ def cli_msg(self) -> str: EmptyLine() HooksRunning(num_hooks=0, hook_type='') HookFinished(stat_line='', execution='') + WriteCatalogFailure(num_exceptions=0) + CatalogWritten(path='') + CannotGenerateDocs() + BuildingCatalog() + diff --git a/core/dbt/task/generate.py b/core/dbt/task/generate.py index de9134ed7fa..e405f54984a 100644 --- a/core/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -16,7 +16,10 @@ ) from dbt.exceptions import InternalException from dbt.include.global_project import DOCS_INDEX_FILE_PATH -from dbt.logger import GLOBAL_LOGGER as logger, print_timestamped_line +from dbt.events.functions import fire_event +from dbt.events.types import ( + WriteCatalogFailure, CatalogWritten, CannotGenerateDocs, BuildingCatalog +) from dbt.parser.manifest import ManifestLoader import dbt.utils import dbt.compilation @@ -203,9 +206,7 @@ def run(self) -> CatalogArtifact: if self.args.compile: compile_results = CompileTask.run(self) if any(r.status == NodeStatus.Error for r in compile_results): - print_timestamped_line( - 'compile failed, cannot generate docs' - ) + fire_event(CannotGenerateDocs()) return CatalogArtifact.from_results( nodes={}, sources={}, @@ -238,7 +239,7 @@ def run(self) -> CatalogArtifact: adapter = get_adapter(self.config) with adapter.connection_named('generate_catalog'): - print_timestamped_line("Building catalog") + fire_event(BuildingCatalog()) catalog_table, exceptions = adapter.get_catalog(self.manifest) catalog_data: List[PrimitiveDict] = [ @@ -267,15 +268,8 @@ def run(self) -> CatalogArtifact: self.write_manifest() if exceptions: - logger.error( - 'dbt encountered {} failure{} while writing the catalog' - .format(len(exceptions), (len(exceptions) != 1) * 's') - ) - - print_timestamped_line( - 'Catalog written to {}'.format(os.path.abspath(path)) - ) - + fire_event(WriteCatalogFailure(num_exceptions=len(exceptions))) + fire_event(CatalogWritten(path=os.path.abspath(path))) return results def get_catalog_results( From 9b805eb226b943ffa1249b28029467655aca3a40 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 11:41:54 -0500 Subject: [PATCH 11/18] added debug level stack trace --- core/dbt/events/types.py | 13 +++++++++++-- core/dbt/task/base.py | 3 ++- core/dbt/task/run_operation.py | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index f875acb97a8..3a6ac19fc6d 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -698,6 +698,7 @@ class InvalidRefInTestNode(WarnLevel, CliEventABC): def cli_msg(self) -> str: return warning_tag(self.msg) class RunningOperationCaughtError(ShowException, ErrorLevel, CliEventABC): +class RunningOperationCaughtError(ErrorLevel, CliEventABC): exc: Exception def cli_msg(self) -> str: @@ -705,7 +706,7 @@ def cli_msg(self) -> str: @dataclass -class RunningOperationUncaughtError(ShowException, ErrorLevel, CliEventABC): +class RunningOperationUncaughtError(ErrorLevel, CliEventABC): exc: Exception def cli_msg(self) -> str: @@ -793,8 +794,16 @@ def cli_msg(self) -> str: ) +# This prints the stack trace at the debug level while allowing just the nice exception message +# at the error level - or whatever other level chosen. Used in multiple places. @dataclass -class GenericExceptionOnRun(ShowException, ErrorLevel, CliEventABC): +class PrintDebugStackTrace(ShowException, DebugLevel, CliEventABC): + def cli_msg(self) -> str: + return "" + + +@dataclass +class GenericExceptionOnRun(ErrorLevel, CliEventABC): build_path: str unique_id: str exc: Exception diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 94b183c0dd3..56ef86c0b3c 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -20,7 +20,7 @@ DbtProjectError, DbtProjectErrorException, DbtProfileError, DbtProfileErrorException, ProfileListTitle, ListSingleProfile, NoDefinedProfiles, ProfileHelpMessage, CatchableExceptionOnRun, InternalExceptionOnRun, GenericExceptionOnRun, - NodeConnectionReleaseError, + NodeConnectionReleaseError, PrintDebugStackTrace ) from .printer import print_skip_caused_by_error, print_skip_line @@ -310,6 +310,7 @@ def _handle_generic_exception(self, e, ctx): fire_event(GenericExceptionOnRun(build_path=self.node.build_path, unique_id=self.node.unique_id, exc=e)) + fire_event(PrintDebugStackTrace()) return str(e) diff --git a/core/dbt/task/run_operation.py b/core/dbt/task/run_operation.py index 35bddfdff63..3c0dfe9c55f 100644 --- a/core/dbt/task/run_operation.py +++ b/core/dbt/task/run_operation.py @@ -12,7 +12,7 @@ from dbt.exceptions import InternalException from dbt.events.functions import fire_event from dbt.events.types import ( - RunningOperationCaughtError, RunningOperationUncaughtError + RunningOperationCaughtError, RunningOperationUncaughtError, PrintDebugStackTrace ) @@ -57,9 +57,11 @@ def run(self) -> RunOperationResultsArtifact: self._run_unsafe() except dbt.exceptions.Exception as exc: fire_event(RunningOperationCaughtError(exc=exc)) + fire_event(PrintDebugStackTrace()) success = False except Exception as exc: fire_event(RunningOperationUncaughtError(exc=exc)) + fire_event(PrintDebugStackTrace()) success = False else: success = True From 03c3b225e872ba302ec951b19265352e9403d9b5 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 10:00:18 -0500 Subject: [PATCH 12/18] fixed flake error --- core/dbt/events/types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 3a6ac19fc6d..458c1998d42 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1102,4 +1102,3 @@ def cli_msg(self) -> str: CatalogWritten(path='') CannotGenerateDocs() BuildingCatalog() - From f3ce43a32f61548b53217578b27cbb97dee9ce8a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 10:09:11 -0500 Subject: [PATCH 13/18] added struct logging to compile --- core/dbt/events/types.py | 6 ++++++ core/dbt/task/compile.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 458c1998d42..a55bb8dfbf0 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -989,6 +989,11 @@ def cli_msg(self) -> str: return "Building catalog" +class CompileComplete(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "Done." + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -1102,3 +1107,4 @@ def cli_msg(self) -> str: CatalogWritten(path='') CannotGenerateDocs() BuildingCatalog() + CompileComplete() diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index 464a4aa8c44..fa328b3789f 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -1,11 +1,14 @@ import threading + +from dbt.contracts.graph.manifest import find_unique_id_for_package from .runnable import GraphRunnableTask from .base import BaseRunner from dbt.contracts.results import RunStatus, RunResult from dbt.exceptions import InternalException from dbt.graph import ResourceTypeSelector -from dbt.logger import print_timestamped_line +from dbt.events.functions import fire_event +from dbt.events.types import CompileComplete from dbt.node_types import NodeType @@ -53,4 +56,4 @@ def get_runner_type(self, _): return CompileRunner def task_end_messages(self, results): - print_timestamped_line('Done.') + fire_event(CompileComplete()) From ae82da2a95015500ad6af3455345a24501fb20e8 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 10:10:36 -0500 Subject: [PATCH 14/18] added struct logging to freshness --- core/dbt/events/types.py | 6 ++++++ core/dbt/task/freshness.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index a55bb8dfbf0..ad12168b97b 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -994,6 +994,11 @@ def cli_msg(self) -> str: return "Done." +class FreshnessCheckComplete(InfoLevel, CliEventABC): + def cli_msg(self) -> str: + return "Done." + + # since mypy doesn't run on every file we need to suggest to mypy that every # class gets instantiated. But we don't actually want to run this code. # making the conditional `if False` causes mypy to skip it as dead code so @@ -1108,3 +1113,4 @@ def cli_msg(self) -> str: CannotGenerateDocs() BuildingCatalog() CompileComplete() + FreshnessCheckComplete() diff --git a/core/dbt/task/freshness.py b/core/dbt/task/freshness.py index a0000399057..e8f765da8c8 100644 --- a/core/dbt/task/freshness.py +++ b/core/dbt/task/freshness.py @@ -16,7 +16,8 @@ SourceFreshnessResult, FreshnessStatus ) from dbt.exceptions import RuntimeException, InternalException -from dbt.logger import print_timestamped_line +from dbt.events.functions import fire_event +from dbt.events.types import FreshnessCheckComplete from dbt.node_types import NodeType from dbt.graph import ResourceTypeSelector @@ -170,4 +171,4 @@ def task_end_messages(self, results): ): print_run_result_error(result) - print_timestamped_line('Done.') + fire_event(FreshnessCheckComplete()) From 4995496d933f75b73dc4a865d5176fece7b13032 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 10:45:56 -0500 Subject: [PATCH 15/18] cleaned up errors --- core/dbt/events/types.py | 11 +++++------ core/dbt/task/compile.py | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index ad12168b97b..fda4e55749d 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -362,8 +362,8 @@ def cli_msg(self) -> str: @dataclass class CatchRunException(ShowException, DebugLevel, CliEventABC): - build_path: Any = '' - exc: Exception = '' + build_path: Any + exc: Exception def cli_msg(self) -> str: INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If the \ @@ -380,7 +380,7 @@ def cli_msg(self) -> str: @dataclass class HandleInternalException(ShowException, DebugLevel, CliEventABC): - exc: Exception = '' + exc: Exception def cli_msg(self) -> str: return str(self.exc) @@ -922,12 +922,11 @@ def cli_msg(self) -> str: @dataclass class DepsNotifyUpdatesAvailable(InfoLevel, CliEventABC): - packages: List + packages: List[str] def cli_msg(self) -> str: return ('\nUpdates available for packages: {} \ - \nUpdate your versions in packages.yml, then run dbt deps', - self.packages) + \nUpdate your versions in packages.yml, then run dbt deps'.format(self.packages)) @dataclass diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index fa328b3789f..dd36e52416d 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -1,6 +1,5 @@ import threading -from dbt.contracts.graph.manifest import find_unique_id_for_package from .runnable import GraphRunnableTask from .base import BaseRunner From 218113b969c305a0c6ac0e7aaf3a36f7781a4db0 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 11:12:45 -0500 Subject: [PATCH 16/18] resolved bug that broke everything --- core/dbt/task/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 56ef86c0b3c..a83f479a74a 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -14,7 +14,7 @@ NotImplementedException, CompilationException, RuntimeException, InternalException ) -from dbt.logger import GLOBAL_LOGGER as log_manager +from dbt.logger import log_manager from dbt.events.functions import fire_event from dbt.events.types import ( DbtProjectError, DbtProjectErrorException, DbtProfileError, DbtProfileErrorException, From 5439c7197354eef1a469000b3e89ab116c79348c Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 11:43:36 -0500 Subject: [PATCH 17/18] removed accidental import --- core/dbt/events/types.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index fda4e55749d..5e60b525c79 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1,9 +1,6 @@ from abc import ABCMeta, abstractmethod from dataclasses import dataclass from typing import Any, List, Optional, Dict -from dbt.ui import warning_tag -import dataclasses -from typing import Any, List, Optional from dbt import ui @@ -688,7 +685,7 @@ class InvalidDisabledSourceInTestNode(WarnLevel, CliEventABC): msg: str def cli_msg(self) -> str: - return warning_tag(self.msg) + return ui.warning_tag(self.msg) @dataclass @@ -696,8 +693,10 @@ class InvalidRefInTestNode(WarnLevel, CliEventABC): msg: str def cli_msg(self) -> str: - return warning_tag(self.msg) -class RunningOperationCaughtError(ShowException, ErrorLevel, CliEventABC): + return ui.warning_tag(self.msg) + + +@dataclass class RunningOperationCaughtError(ErrorLevel, CliEventABC): exc: Exception From 84a89197c8e4c53a976a9bbe00b8dd731ad3a56f Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 2 Nov 2021 13:26:40 -0500 Subject: [PATCH 18/18] fixed bug with unused args --- core/dbt/events/types.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 5e60b525c79..2827fc3d95e 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -903,10 +903,7 @@ def cli_msg(self) -> str: return f" Updated version available: {self.version_latest}" -@dataclass class DepsUTD(InfoLevel, CliEventABC): - version_latest: str - def cli_msg(self) -> str: return " Up to date!"