From 08b2d94ccd7904879c062ed801ea61fdda0d2603 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Thu, 26 Jan 2023 10:42:49 -0800 Subject: [PATCH] =?UTF-8?q?[CT-920][CT-1900]=20Create=20Click=20CLI=20runn?= =?UTF-8?q?er=20and=20use=20it=20to=20fix=20dbt=20docs=20=E2=80=A6=20(#672?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Github Build Bot --- .../Under the Hood-20230125-041136.yaml | 6 + core/dbt/cli/main.py | 23 +++- core/dbt/events/proto_types.py | 24 ---- core/dbt/events/types.py | 27 ---- core/dbt/task/serve.py | 46 ++----- tests/functional/minimal_cli/fixtures.py | 120 ++++++++++++++++++ .../minimal_cli/test_minimal_cli.py | 49 +++++++ tests/unit/test_events.py | 32 ++--- 8 files changed, 218 insertions(+), 109 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20230125-041136.yaml create mode 100644 tests/functional/minimal_cli/fixtures.py create mode 100644 tests/functional/minimal_cli/test_minimal_cli.py diff --git a/.changes/unreleased/Under the Hood-20230125-041136.yaml b/.changes/unreleased/Under the Hood-20230125-041136.yaml new file mode 100644 index 00000000000..ff02d3ef752 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230125-041136.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: '[CT-920][CT-1900] Create Click CLI runner and use it to fix dbt docs commands' +time: 2023-01-25T04:11:36.57506-08:00 +custom: + Author: aranke + Issue: 5544 6722 diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index f5ea1095cc7..272334fe303 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -13,6 +13,7 @@ from dbt.task.deps import DepsTask from dbt.task.debug import DebugTask from dbt.task.run import RunTask +from dbt.task.serve import ServeTask from dbt.task.test import TestTask from dbt.task.snapshot import SnapshotTask from dbt.task.seed import SeedTask @@ -172,6 +173,7 @@ def docs(ctx, **kwargs): @p.models @p.profile @p.profiles_dir +@p.project_dir @p.select @p.selector @p.state @@ -187,7 +189,11 @@ def docs(ctx, **kwargs): @requires.manifest def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" - task = GenerateTask(ctx.obj["flags"], ctx.obj["runtime_config"]) + task = GenerateTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) results = task.run() success = task.interpret_results(results) @@ -205,10 +211,21 @@ def docs_generate(ctx, **kwargs): @p.target @p.vars @requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {ctx.obj['flags']}") - return None, True + task = ServeTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt compile diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 5ee384643d3..37fe69453f6 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1971,30 +1971,6 @@ class EmptyLine(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) -@dataclass -class ServingDocsPort(betterproto.Message): - """Z018""" - - info: "EventInfo" = betterproto.message_field(1) - address: str = betterproto.string_field(2) - port: int = betterproto.int32_field(3) - - -@dataclass -class ServingDocsAccessInfo(betterproto.Message): - """Z019""" - - info: "EventInfo" = betterproto.message_field(1) - port: str = betterproto.string_field(2) - - -@dataclass -class ServingDocsExitInfo(betterproto.Message): - """Z020""" - - info: "EventInfo" = betterproto.message_field(1) - - @dataclass class RunResultWarning(betterproto.Message): """Z021""" diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 0a0cd04fe1d..f56aaf51e4e 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -2463,33 +2463,6 @@ def message(self) -> str: return "" -@dataclass -class ServingDocsPort(InfoLevel, pt.ServingDocsPort): - def code(self): - return "Z018" - - def message(self) -> str: - return f"Serving docs at {self.address}:{self.port}" - - -@dataclass -class ServingDocsAccessInfo(InfoLevel, pt.ServingDocsAccessInfo): - def code(self): - return "Z019" - - def message(self) -> str: - return f"To access from your browser, navigate to: http://localhost:{self.port}" - - -@dataclass -class ServingDocsExitInfo(InfoLevel, pt.ServingDocsExitInfo): - def code(self): - return "Z020" - - def message(self) -> str: - return "Press Ctrl+C to exit." - - @dataclass class RunResultWarning(WarnLevel, pt.RunResultWarning): def code(self): diff --git a/core/dbt/task/serve.py b/core/dbt/task/serve.py index 4d702234d0e..696be89a37f 100644 --- a/core/dbt/task/serve.py +++ b/core/dbt/task/serve.py @@ -1,46 +1,28 @@ -import shutil import os +import shutil +import socketserver import webbrowser - -from dbt.include.global_project import DOCS_INDEX_FILE_PATH from http.server import SimpleHTTPRequestHandler -from socketserver import TCPServer -from dbt.events.functions import fire_event -from dbt.events.types import ServingDocsPort, ServingDocsAccessInfo, ServingDocsExitInfo, EmptyLine +import click + +from dbt.include.global_project import DOCS_INDEX_FILE_PATH from dbt.task.base import ConfiguredTask class ServeTask(ConfiguredTask): def run(self): os.chdir(self.config.target_path) - - port = self.args.port - address = "0.0.0.0" - shutil.copyfile(DOCS_INDEX_FILE_PATH, "index.html") - fire_event(ServingDocsPort(address=address, port=port)) - fire_event(ServingDocsAccessInfo(port=port)) - fire_event(EmptyLine()) - fire_event(EmptyLine()) - fire_event(ServingDocsExitInfo()) - - # mypy doesn't think SimpleHTTPRequestHandler is ok here, but it is - httpd = TCPServer( # type: ignore - (address, port), SimpleHTTPRequestHandler # type: ignore - ) # type: ignore - - if self.args.open_browser: - try: - webbrowser.open_new_tab(f"http://127.0.0.1:{port}") - except webbrowser.Error: - pass + port = self.args.port - try: - httpd.serve_forever() # blocks - finally: - httpd.shutdown() - httpd.server_close() + if self.args.browser: + webbrowser.open_new_tab(f"http://localhost:{port}") - return None + with socketserver.TCPServer(("", port), SimpleHTTPRequestHandler) as httpd: + click.echo(f"Serving docs at {port}") + click.echo(f"To access from your browser, navigate to: http://localhost:{port}") + click.echo("\n\n") + click.echo("Press Ctrl+C to exit.") + httpd.serve_forever() diff --git a/tests/functional/minimal_cli/fixtures.py b/tests/functional/minimal_cli/fixtures.py new file mode 100644 index 00000000000..ac746389c6d --- /dev/null +++ b/tests/functional/minimal_cli/fixtures.py @@ -0,0 +1,120 @@ +import pytest + +models__schema_yml = """ +version: 2 +models: + - name: sample_model + columns: + - name: sample_num + tests: + - accepted_values: + values: [1, 2] + - not_null + - name: sample_bool + tests: + - not_null + - unique +""" + +models__sample_model = """ +select * from {{ ref('sample_seed') }} +""" + +snapshots__sample_snapshot = """ +{% snapshot orders_snapshot %} + +{{ + config( + target_database='postgres', + target_schema='snapshots', + unique_key='sample_num', + strategy='timestamp', + updated_at='updated_at', + ) +}} + +select * from {{ ref('sample_model') }} + +{% endsnapshot %} +""" + +seeds__sample_seed = """sample_num,sample_bool +1,true +2,false +,true +""" + +tests__failing_sql = """ +{{ config(severity = 'warn') }} +select 1 +""" + + +class BaseConfigProject: + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "jaffle_shop", + "profile": "jaffle_shop", + "version": "0.1.0", + "config-version": 2, + "clean-targets": [ + "target", + "dbt_packages", + "logs" + ] + } + + @pytest.fixture(scope="class") + def profiles_config_update(self): + return { + "jaffle_shop": { + "outputs": { + "dev": { + "type": "postgres", + "database": "postgres", + "schema": "jaffle_shop", + "host": "localhost", + "user": "root", + "port": 5432, + "password": "password" + } + }, + "target": "dev" + } + } + + @pytest.fixture(scope="class") + def packages(self): + return { + "packages": [ + { + "package": "dbt-labs/dbt_utils", + "version": "1.0.0" + } + ] + } + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "sample_model.sql": models__sample_model, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return { + "sample_snapshot.sql": snapshots__sample_snapshot + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"sample_seed.csv": seeds__sample_seed} + + @pytest.fixture(scope="class") + def tests(self): + return { + "failing.sql": tests__failing_sql, + } diff --git a/tests/functional/minimal_cli/test_minimal_cli.py b/tests/functional/minimal_cli/test_minimal_cli.py new file mode 100644 index 00000000000..a87c0c95f93 --- /dev/null +++ b/tests/functional/minimal_cli/test_minimal_cli.py @@ -0,0 +1,49 @@ +import pytest +from click.testing import CliRunner + +from dbt.cli.main import cli +from tests.functional.minimal_cli.fixtures import BaseConfigProject + + +class TestMinimalCli(BaseConfigProject): + """Test the minimal/happy-path for the CLI using the Click CliRunner""" + @pytest.fixture(scope="class") + def runner(self): + return CliRunner() + + def test_clean(self, runner, project): + result = runner.invoke(cli, ['clean']) + assert 'target' in result.output + assert 'dbt_packages' in result.output + assert 'logs' in result.output + + def test_deps(self, runner, project): + result = runner.invoke(cli, ['deps']) + assert 'dbt-labs/dbt_utils' in result.output + assert '1.0.0' in result.output + + def test_ls(self, runner, project): + runner.invoke(cli, ['deps']) + ls_result = runner.invoke(cli, ['ls']) + assert '1 seed' in ls_result.output + assert '1 model' in ls_result.output + assert '5 tests' in ls_result.output + assert '1 snapshot' in ls_result.output + + def test_build(self, runner, project): + runner.invoke(cli, ['deps']) + result = runner.invoke(cli, ['build']) + # 1 seed, 1 model, 2 tests + assert 'PASS=4' in result.output + # 2 tests + assert 'ERROR=2' in result.output + # Singular test + assert 'WARN=1' in result.output + # 1 snapshot + assert 'SKIP=1' in result.output + + def test_docs_generate(self, runner, project): + runner.invoke(cli, ['deps']) + result = runner.invoke(cli, ['docs', 'generate']) + assert 'Building catalog' in result.output + assert 'Catalog written' in result.output diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 3dbff04c303..935c3421607 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -1,29 +1,18 @@ # flake8: noqa -from dbt.events.test_types import UnitTestInfo -from dbt.events import AdapterLogger -from dbt.events.functions import event_to_json, LOG_VERSION, event_to_dict -from dbt.events.types import * -from dbt.events.test_types import * +import re +from typing import TypeVar +from dbt.contracts.files import FileHash +from dbt.contracts.graph.nodes import ModelNode, NodeConfig, DependsOn +from dbt.events import AdapterLogger from dbt.events.base_types import ( BaseEvent, - DebugLevel, - WarnLevel, - InfoLevel, - ErrorLevel, TestLevel, ) -from dbt.events.proto_types import ListOfStrings, NodeInfo, RunResultMsg, ReferenceKeyMsg -from importlib import reload -import dbt.events.functions as event_funcs -import dbt.flags as flags -import inspect -import json -from dbt.contracts.graph.nodes import ModelNode, NodeConfig, DependsOn -from dbt.contracts.files import FileHash -from mashumaro.types import SerializableType -from typing import Generic, TypeVar, Dict -import re +from dbt.events.functions import event_to_json, event_to_dict +from dbt.events.test_types import * +from dbt.events.types import * + # takes in a class and finds any subclasses for it def get_all_subclasses(cls): @@ -459,9 +448,6 @@ def MockNode(): FinishedCleanPaths(), OpenCommand(open_cmd="", profiles_dir=""), EmptyLine(), - ServingDocsPort(address="", port=0), - ServingDocsAccessInfo(port=""), - ServingDocsExitInfo(), RunResultWarning(resource_type="", node_name="", path=""), RunResultFailure(resource_type="", node_name="", path=""), StatsLine(stats={"error": 0, "skip": 0, "pass": 0, "warn": 0,"total": 0}),