From 212c36608cfa384300b1c82807e88ba8a9af2d50 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Fri, 20 Jan 2023 17:11:38 -0500 Subject: [PATCH 1/2] CT-1718: Add Note and Formatting event types --- core/dbt/events/functions.py | 6 +++--- core/dbt/events/proto_types.py | 34 +++++++++++++++++----------------- core/dbt/events/types.proto | 27 +++++++++++++++------------ core/dbt/events/types.py | 29 ++++++++++++++++++----------- core/dbt/task/deps.py | 4 ++-- core/dbt/task/printer.py | 12 ++++++------ core/dbt/task/run.py | 8 ++++---- core/dbt/task/runnable.py | 8 ++++---- core/dbt/task/seed.py | 9 ++++----- core/dbt/task/serve.py | 11 ++++++++--- tests/unit/test_events.py | 4 ++-- 11 files changed, 83 insertions(+), 69 deletions(-) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index f32287c3049..786e9cdf91d 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -3,7 +3,7 @@ from dbt.events.base_types import BaseEvent, Cache, EventLevel, NoFile, NoStdOut, EventMsg from dbt.events.eventmgr import EventManager, LoggerConfig, LineFormat, NoFilter from dbt.events.helpers import env_secrets, scrub_secrets -from dbt.events.types import EmptyLine +from dbt.events.types import Formatting import dbt.flags as flags from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing from functools import partial @@ -65,7 +65,7 @@ def _stdout_filter( and (not isinstance(msg.data, Cache) or log_cache_events) and (EventLevel(msg.info.level) != EventLevel.DEBUG or debug_mode) and (EventLevel(msg.info.level) == EventLevel.ERROR or not quiet_mode) - and not (flags.LOG_FORMAT == "json" and type(msg.data) == EmptyLine) + and not (flags.LOG_FORMAT == "json" and type(msg.data) == Formatting) ) @@ -85,7 +85,7 @@ def _logfile_filter(log_cache_events: bool, msg: EventMsg) -> bool: return ( not isinstance(msg.data, NoFile) and not (isinstance(msg.data, Cache) and not log_cache_events) - and not (flags.LOG_FORMAT == "json" and type(msg.data) == EmptyLine) + and not (flags.LOG_FORMAT == "json" and type(msg.data) == Formatting) ) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 3fb92eeda51..2b47a689057 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1859,19 +1859,6 @@ class SeedHeaderMsg(betterproto.Message): data: "SeedHeader" = betterproto.message_field(2) -@dataclass -class SeedHeaderSeparator(betterproto.Message): - """Q005""" - - len_header: int = betterproto.int32_field(1) - - -@dataclass -class SeedHeaderSeparatorMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "SeedHeaderSeparator" = betterproto.message_field(2) - - @dataclass class SQLRunnerException(betterproto.Message): """Q006""" @@ -2511,16 +2498,16 @@ class OpenCommandMsg(betterproto.Message): @dataclass -class EmptyLine(betterproto.Message): +class Formatting(betterproto.Message): """Z017""" - pass + msg: str = betterproto.string_field(1) @dataclass -class EmptyLineMsg(betterproto.Message): +class FormattingMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "EmptyLine" = betterproto.message_field(2) + data: "Formatting" = betterproto.message_field(2) @dataclass @@ -2847,6 +2834,19 @@ class RunResultWarningMessageMsg(betterproto.Message): data: "RunResultWarningMessage" = betterproto.message_field(2) +@dataclass +class Note(betterproto.Message): + """Z047""" + + msg: str = betterproto.string_field(1) + + +@dataclass +class NoteMsg(betterproto.Message): + info: "EventInfo" = betterproto.message_field(1) + data: "Note" = betterproto.message_field(2) + + @dataclass class IntegrationTestInfo(betterproto.Message): """T001""" diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 87a10c19eda..938f97dfd02 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1473,15 +1473,7 @@ message SeedHeaderMsg { SeedHeader data = 2; } -// Q005 -message SeedHeaderSeparator { - int32 len_header = 1; -} - -message SeedHeaderSeparatorMsg { - EventInfo info = 1; - SeedHeaderSeparator data = 2; -} +// Skipped Q005 // Q006 message SQLRunnerException { @@ -2004,12 +1996,13 @@ message OpenCommandMsg { } // Z017 -message EmptyLine { +message Formatting { + string msg = 1; } -message EmptyLineMsg { +message FormattingMsg { EventInfo info = 1; - EmptyLine data = 2; + Formatting data = 2; } // Z018 @@ -2258,6 +2251,16 @@ message RunResultWarningMessageMsg { RunResultWarningMessage data = 2; } +// Z047 +message Note { + string msg = 1; +} + +message NoteMsg { + EventInfo info = 1; + Note data = 2; +} + // T - Integration tests // T001 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 043ed8d40ff..cbae5fe354d 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1498,15 +1498,6 @@ def message(self) -> str: return self.header -@dataclass -class SeedHeaderSeparator(InfoLevel, pt.SeedHeaderSeparator): - def code(self): - return "Q005" - - def message(self) -> str: - return "-" * self.len_header - - @dataclass class SQLRunnerException(DebugLevel, pt.SQLRunnerException): # noqa def code(self): @@ -2084,13 +2075,18 @@ def message(self) -> str: return msg +# We use events to create console output, but also think of them as a sequence of important and +# meaningful occurrences to be used for debugging and monitoring. The Formatting event helps eases +# the tension between these two goals by allowing empty lines, heading separators, and other +# formatting to be written to the console, while they can be ignored for other purposes. For +# general information that isn't simple formatting, the Note event should be used instead. @dataclass -class EmptyLine(InfoLevel, pt.EmptyLine): +class Formatting(InfoLevel, pt.Formatting): def code(self): return "Z017" def message(self) -> str: - return "" + return self.msg @dataclass @@ -2345,3 +2341,14 @@ def code(self): def message(self) -> str: # This is the message on the result object, cannot be formatted in event return self.msg + + +# The Note event provides a way to log messages which aren't likely to be useful as more structured events. +# For conslole formatting text like empty lines and separator bars, use the Formatting event instead. +@dataclass +class Note(InfoLevel, pt.Note): + def code(self): + return "Z047" + + def message(self) -> str: + return self.msg diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 0052840c570..ac6a6c41af3 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -20,7 +20,7 @@ DepsInstallInfo, DepsListSubdirectory, DepsNotifyUpdatesAvailable, - EmptyLine, + Formatting, ) from dbt.clients import system @@ -88,7 +88,7 @@ def run(self) -> None: package_name=package_name, source_type=source_type, version=version ) if packages_to_upgrade: - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(DepsNotifyUpdatesAvailable(packages=ListOfStrings(packages_to_upgrade))) @classmethod diff --git a/core/dbt/task/printer.py b/core/dbt/task/printer.py index edb2592d194..9fae854bdb4 100644 --- a/core/dbt/task/printer.py +++ b/core/dbt/task/printer.py @@ -5,7 +5,7 @@ ) from dbt.events.functions import fire_event from dbt.events.types import ( - EmptyLine, + Formatting, RunResultWarning, RunResultWarningMessage, RunResultFailure, @@ -72,14 +72,14 @@ def print_run_status_line(results) -> None: stats["total"] += 1 with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(StatsLine(stats=stats)) def print_run_result_error(result, newline: bool = True, is_warning: bool = False) -> None: if newline: with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) if result.status == NodeStatus.Fail or (is_warning and result.status == NodeStatus.Warn): if is_warning: @@ -109,12 +109,12 @@ def print_run_result_error(result, newline: bool = True, is_warning: bool = Fals if result.node.build_path is not None: with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(SQLCompiledPath(path=result.node.compiled_path)) if result.node.should_store_failures: with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(CheckNodeTestFailure(relation_name=result.node.relation_name)) elif result.message is not None: @@ -143,7 +143,7 @@ def print_run_end_messages(results, keyboard_interrupt: bool = False) -> None: with DbtStatusMessage(), InvocationProcessor(): with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event( EndOfRunSummary( num_errors=len(errors), diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 411c57af663..fd24dd06ba2 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -30,7 +30,7 @@ from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import ( DatabaseErrorRunningHook, - EmptyLine, + Formatting, HooksRunning, FinishedRunningStats, LogModelResult, @@ -335,7 +335,7 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): num_hooks = len(ordered_hooks) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(HooksRunning(num_hooks=num_hooks, hook_type=hook_type)) startctx = TimestampNamed("node_started_at") @@ -388,7 +388,7 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): self._total_executed += len(ordered_hooks) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) def safe_run_hooks( self, adapter, hook_type: RunHookType, extra_context: Dict[str, Any] @@ -419,7 +419,7 @@ def print_results_line(self, results, execution_time): execution = utils.humanize_execution_time(execution_time=execution_time) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event( FinishedRunningStats( stat_line=stat_line, execution=execution, execution_time=execution_time diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index fee5fadc891..fd383226770 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -28,7 +28,7 @@ ) from dbt.events.functions import fire_event, warn_or_error from dbt.events.types import ( - EmptyLine, + Formatting, LogCancelLine, DefaultSelector, NodeStart, @@ -377,7 +377,7 @@ def execute_nodes(self): ) ) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) pool = ThreadPool(num_threads) try: @@ -458,7 +458,7 @@ def run(self): if len(self._flattened_nodes) == 0: with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) warn_or_error(NothingToDo()) result = self.get_result( results=[], @@ -467,7 +467,7 @@ def run(self): ) else: with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) selected_uids = frozenset(n.unique_id for n in self._flattened_nodes) result = self.execute_with_hooks(selected_uids) diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 58b6aa25bda..9ec1df3b81f 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -12,8 +12,7 @@ from dbt.events.functions import fire_event from dbt.events.types import ( SeedHeader, - SeedHeaderSeparator, - EmptyLine, + Formatting, LogSeedResult, LogStartLine, ) @@ -99,13 +98,13 @@ def show_table(self, result): header = "Random sample of table: {}.{}".format(schema, alias) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) fire_event(SeedHeader(header=header)) - fire_event(SeedHeaderSeparator(len_header=len(header))) + fire_event(Formatting("-" * len(header))) rand_table.print_table(max_rows=10, max_columns=None) with TextOnly(): - fire_event(EmptyLine()) + fire_event(Formatting("")) def show_tables(self, results): for result in results: diff --git a/core/dbt/task/serve.py b/core/dbt/task/serve.py index 4d702234d0e..cbdc48a8010 100644 --- a/core/dbt/task/serve.py +++ b/core/dbt/task/serve.py @@ -6,7 +6,12 @@ from http.server import SimpleHTTPRequestHandler from socketserver import TCPServer from dbt.events.functions import fire_event -from dbt.events.types import ServingDocsPort, ServingDocsAccessInfo, ServingDocsExitInfo, EmptyLine +from dbt.events.types import ( + ServingDocsPort, + ServingDocsAccessInfo, + ServingDocsExitInfo, + Formatting, +) from dbt.task.base import ConfiguredTask @@ -22,8 +27,8 @@ def run(self): fire_event(ServingDocsPort(address=address, port=port)) fire_event(ServingDocsAccessInfo(port=port)) - fire_event(EmptyLine()) - fire_event(EmptyLine()) + fire_event(Formatting("")) + fire_event(Formatting("")) fire_event(ServingDocsExitInfo()) # mypy doesn't think SimpleHTTPRequestHandler is ok here, but it is diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 5e412e34f33..ca866520afe 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -261,7 +261,6 @@ def test_event_codes(self): CompileComplete(), FreshnessCheckComplete(), SeedHeader(header=""), - SeedHeaderSeparator(len_header=0), SQLRunnerException(exc=""), LogTestResult( name="", @@ -358,7 +357,7 @@ def test_event_codes(self): ProtectedCleanPath(path=""), FinishedCleanPaths(), OpenCommand(open_cmd="", profiles_dir=""), - EmptyLine(), + Formatting(msg=""), ServingDocsPort(address="", port=0), ServingDocsAccessInfo(port=""), ServingDocsExitInfo(), @@ -383,6 +382,7 @@ def test_event_codes(self): FlushEventsFailure(), TrackingInitializeFailure(), RunResultWarningMessage(), + Note(msg="This is a note."), # T - tests ====================== IntegrationTestInfo(), IntegrationTestDebug(), From 301c67ba53dccd26269fe344dc9983463c6810eb Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Fri, 20 Jan 2023 17:23:29 -0500 Subject: [PATCH 2/2] CT-1718: Add changelog entry --- .changes/unreleased/Under the Hood-20230120-172254.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230120-172254.yaml diff --git a/.changes/unreleased/Under the Hood-20230120-172254.yaml b/.changes/unreleased/Under the Hood-20230120-172254.yaml new file mode 100644 index 00000000000..3f65b39f99e --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230120-172254.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Replaced the EmptyLine event with a more general Formatting event, and added + a Note event. +time: 2023-01-20T17:22:54.45828-05:00 +custom: + Author: peterallenwebb + Issue: "6481"