From d4e6d796822b99ff43ebd4e5a82ee7765e21b97f Mon Sep 17 00:00:00 2001 From: Eli Kastelein Date: Fri, 5 Mar 2021 14:15:53 -0800 Subject: [PATCH 1/7] Check if column exists when altering column comments in snowflake --- plugins/snowflake/dbt/include/snowflake/macros/adapters.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql index 8ee794734db..21ad187fb2b 100644 --- a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql +++ b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql @@ -155,9 +155,8 @@ {% macro snowflake__alter_column_comment(relation, column_dict) -%} - alter {{ relation.type }} {{ relation }} alter {% for column_name in column_dict %} - {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} COMMENT $${{ column_dict[column_name]['description'] | replace('$', '[$]') }}$$ {{ ',' if not loop.last else ';' }} + comment if exists on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is $${{ column_dict[column_name]['description'] | replace('$', '[$]') }}$$; {% endfor %} {% endmacro %} From 6f2ef062770a63786b79ee3812a0cdc2ff223701 Mon Sep 17 00:00:00 2001 From: Eli Kastelein Date: Wed, 12 May 2021 16:19:30 -0700 Subject: [PATCH 2/7] Add new test class for persist docs models with missing columns --- .../models-column-missing/missing_column.sql | 2 ++ .../models-column-missing/schema.yml | 8 +++++ .../test_persist_docs.py | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql create mode 100644 test/integration/060_persist_docs_tests/models-column-missing/schema.yml diff --git a/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql b/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql new file mode 100644 index 00000000000..642b0f14a19 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models-column-missing/missing_column.sql @@ -0,0 +1,2 @@ +{{ config(materialized='table') }} +select 1 as id, 'Ed' as name diff --git a/test/integration/060_persist_docs_tests/models-column-missing/schema.yml b/test/integration/060_persist_docs_tests/models-column-missing/schema.yml new file mode 100644 index 00000000000..aa7b4f88820 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models-column-missing/schema.yml @@ -0,0 +1,8 @@ +version: 2 +models: + - name: missing_column + columns: + - name: id + description: "test id column description" + - name: column_that_does_not_exist + description: "comment that cannot be created" diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py index 986bfe6fc2d..116818c0df9 100644 --- a/test/integration/060_persist_docs_tests/test_persist_docs.py +++ b/test/integration/060_persist_docs_tests/test_persist_docs.py @@ -284,3 +284,38 @@ def test_bigquery_persist_docs(self): level_3_column = node['columns']['level_1.level_2.level_3_a'] assert level_3_column['comment'] == "level_3 column description" + + +class TestPersistDocsColumnMissing(BasePersistDocsTest): + @property + def project_config(self): + return { + 'config-version': 2, + 'models': { + 'test': { + '+persist_docs': { + "columns": True, + }, + } + } + } + + @property + def models(self): + return 'models-column-missing' + + @use_profile('snowflake') + def test_snowflake_missing_column(self): + self.run_dbt() + self.run_dbt(['docs', 'generate']) + with open('target/catalog.json') as fp: + catalog_data = json.load(fp) + assert 'nodes' in catalog_data + + table_node = catalog_data['nodes']['model.test.missing_column'] + table_id_comment = table_node['columns']['ID']['comment'] + assert table_id_comment.startswith('test id column description') + + @use_profile('bigquery') + def test_bigquery_missing_column(self): + self.run_dbt() From 7cfd2b37ceeaa70a46d7d927b6296521339d07b5 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 11 May 2021 13:23:38 -0400 Subject: [PATCH 3/7] Parallel run all integration tests after unit (#3328) --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ed68438b92..c5e7bef258c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -62,7 +62,7 @@ jobs: - job: SnowflakeIntegrationTest pool: vmImage: 'vs2017-win2016' - dependsOn: PostgresIntegrationTest + dependsOn: UnitTest condition: succeeded() steps: - task: UsePythonVersion@0 @@ -87,7 +87,7 @@ jobs: - job: BigQueryIntegrationTest pool: vmImage: 'vs2017-win2016' - dependsOn: PostgresIntegrationTest + dependsOn: UnitTest condition: succeeded() steps: - task: UsePythonVersion@0 @@ -104,7 +104,7 @@ jobs: - job: RedshiftIntegrationTest pool: vmImage: 'vs2017-win2016' - dependsOn: PostgresIntegrationTest + dependsOn: UnitTest condition: succeeded() steps: - task: UsePythonVersion@0 From 1127ad689c2badeffdc0098a433661318cbaa67f Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Tue, 11 May 2021 10:45:16 -0400 Subject: [PATCH 4/7] don't clobber default args --- core/dbt/clients/jinja.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 2c8c7f7877f..be0c101f47f 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -416,7 +416,6 @@ def parse(self, parser): test_name = parser.parse_assign_target(name_only=True).name parser.parse_signature(node) - node.defaults = [] node.name = get_test_macro_name(test_name) node.body = parser.parse_statements(('name:endtest',), drop_needle=True) From 695c1383fa47e35bf0d63d941c7b53e7ba16be58 Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Wed, 12 May 2021 09:54:20 -0400 Subject: [PATCH 5/7] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76076a9ac92..bba596854ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Fixes - Fix compiled sql for ephemeral models ([#3317](https://github.com/fishtown-analytics/dbt/issues/3317), [#3318](https://github.com/fishtown-analytics/dbt/pull/3318)) - Now generating `run_results.json` even when no nodes are selected ([#3313](https://github.com/fishtown-analytics/dbt/issues/3313), [#3315](https://github.com/fishtown-analytics/dbt/pull/3315)) +- Stop clobbering default keyword arguments for jinja test definitions ([#3329](https://github.com/fishtown-analytics/dbt/issues/3329), [#3340](https://github.com/fishtown-analytics/dbt/pull/3340)) ### Under the hood - Added logic for registry requests to raise a timeout error after a response hangs out for 30 seconds and 5 attempts have been made to reach the endpoint ([#3177](https://github.com/fishtown-analytics/dbt/issues/3177), [#3275](https://github.com/fishtown-analytics/dbt/pull/3275)) From 3f18907eb62ae846626944bdd7d0e4249066afa2 Mon Sep 17 00:00:00 2001 From: Eli Kastelein Date: Wed, 12 May 2021 16:50:51 -0700 Subject: [PATCH 6/7] Update changelog for PR #3149 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bba596854ed..a4849e9e112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Fix compiled sql for ephemeral models ([#3317](https://github.com/fishtown-analytics/dbt/issues/3317), [#3318](https://github.com/fishtown-analytics/dbt/pull/3318)) - Now generating `run_results.json` even when no nodes are selected ([#3313](https://github.com/fishtown-analytics/dbt/issues/3313), [#3315](https://github.com/fishtown-analytics/dbt/pull/3315)) - Stop clobbering default keyword arguments for jinja test definitions ([#3329](https://github.com/fishtown-analytics/dbt/issues/3329), [#3340](https://github.com/fishtown-analytics/dbt/pull/3340)) +- Change the snowflake adapter to only comment on a column if it exists when using the persist_docs config ([#3039](https://github.com/fishtown-analytics/dbt/issues/3039)) ### Under the hood - Added logic for registry requests to raise a timeout error after a response hangs out for 30 seconds and 5 attempts have been made to reach the endpoint ([#3177](https://github.com/fishtown-analytics/dbt/issues/3177), [#3275](https://github.com/fishtown-analytics/dbt/pull/3275)) @@ -11,6 +12,7 @@ Contributors: - [@TeddyCr](https://github.com/TeddyCr) ([#3275](https://github.com/fishtown-analytics/dbt/pull/3275)) - [@panasenco](https://github.com/panasenco) ([#3315](https://github.com/fishtown-analytics/dbt/pull/3315)) +- [@elikastelein](https://github.com/elikastelein) ([#3149](https://github.com/fishtown-analytics/dbt/pull/3149)) ## dbt 0.20.0b1 (May 03, 2021) From 0c15a9a2d2a7864a6246e32a300e4241c1fc47ee Mon Sep 17 00:00:00 2001 From: Eli Kastelein Date: Wed, 12 May 2021 17:13:37 -0700 Subject: [PATCH 7/7] Pull in upstream changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4849e9e112..cfbbf57d919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Fix compiled sql for ephemeral models ([#3317](https://github.com/fishtown-analytics/dbt/issues/3317), [#3318](https://github.com/fishtown-analytics/dbt/pull/3318)) - Now generating `run_results.json` even when no nodes are selected ([#3313](https://github.com/fishtown-analytics/dbt/issues/3313), [#3315](https://github.com/fishtown-analytics/dbt/pull/3315)) - Stop clobbering default keyword arguments for jinja test definitions ([#3329](https://github.com/fishtown-analytics/dbt/issues/3329), [#3340](https://github.com/fishtown-analytics/dbt/pull/3340)) -- Change the snowflake adapter to only comment on a column if it exists when using the persist_docs config ([#3039](https://github.com/fishtown-analytics/dbt/issues/3039)) +- Update the snowflake adapter to only comment on a column if it exists when using the persist_docs config ([#3039](https://github.com/fishtown-analytics/dbt/issues/3039)) ### Under the hood - Added logic for registry requests to raise a timeout error after a response hangs out for 30 seconds and 5 attempts have been made to reach the endpoint ([#3177](https://github.com/fishtown-analytics/dbt/issues/3177), [#3275](https://github.com/fishtown-analytics/dbt/pull/3275))