Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BigQuery Labels #1964

Merged
merged 6 commits into from
Dec 13, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/bigquery/dbt/adapters/bigquery/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class BigQueryAdapter(BaseAdapter):
ConnectionManager = BigQueryConnectionManager

AdapterSpecificConfigs = frozenset({"cluster_by", "partition_by",
"kms_key_name"})
"kms_key_name", "labels"})

###
# Implementations of abstract methods
Expand Down
40 changes: 25 additions & 15 deletions plugins/bigquery/dbt/include/bigquery/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -26,49 +26,59 @@

{%- endmacro -%}

{% macro bigquery_table_options(persist_docs, temporary, kms_key_name) %}
{% set opts = {} %}
{% macro bigquery_table_options(persist_docs, temporary, kms_key_name, labels) %}
{% set opts = {} -%}
kconvey marked this conversation as resolved.
Show resolved Hide resolved

{% set description = get_relation_comment(persist_docs, model) %}
{%- set description = get_relation_comment(persist_docs, model) -%}
{%- if description is not none -%}
{% do opts.update({'description': "'" ~ description ~ "'"}) %}
{% endif %}
{% if temporary %}
{%- do opts.update({'description': "'" ~ description ~ "'"}) -%}
{%- endif -%}
{%- if temporary -%}
{% do opts.update({'expiration_timestamp': 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'}) %}
{% endif %}
{% if kms_key_name %}
{% do opts.update({'kms_key_name': "'" ~ kms_key_name ~ "'"}) %}
{% endif %}
{%- endif -%}
{%- if kms_key_name -%}
{%- do opts.update({'kms_key_name': "'" ~ kms_key_name ~ "'"}) -%}
{%- endif -%}
{%- if labels -%}
{%- set label_list = [] -%}
{%- for label, value in labels.items() -%}
{%- do label_list.append((label, value)) -%}
{%- endfor -%}
{%- do opts.update({'labels': label_list}) -%}
{%- endif -%}

{% set options -%}
OPTIONS({% for opt_key, opt_val in opts.items() %}
{{ opt_key }}={{ opt_val }}{{ "," if not loop.last }}
{% endfor %})
{%- endset %}
{% do return(options) %}
{%- do return(options) -%}
{%- endmacro -%}

{% macro bigquery__create_table_as(temporary, relation, sql) -%}
{%- set raw_partition_by = config.get('partition_by', none) -%}
{%- set raw_cluster_by = config.get('cluster_by', none) -%}
{%- set raw_persist_docs = config.get('persist_docs', {}) -%}
{%- set raw_kms_key_name = config.get('kms_key_name', none) -%}

{%- set raw_labels = config.get('labels', []) -%}
create or replace table {{ relation }}
{{ partition_by(raw_partition_by) }}
{{ cluster_by(raw_cluster_by) }}
{{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=temporary, kms_key_name=raw_kms_key_name) }}
{{ bigquery_table_options(
persist_docs=raw_persist_docs, temporary=temporary, kms_key_name=raw_kms_key_name,
labels=raw_labels) }}
as (
{{ sql }}
);
{% endmacro %}
{%- endmacro -%}


{% macro bigquery__create_view_as(relation, sql) -%}
{%- set raw_persist_docs = config.get('persist_docs', {}) -%}
{%- set raw_labels = config.get('labels', []) -%}

create or replace view {{ relation }}
{{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=false) }}
{{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=false, labels=raw_labels) }}
as (
{{ sql }}
);
Expand Down
8 changes: 8 additions & 0 deletions test/integration/022_bigquery_test/models/labeled_model.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{{
config(
materialized = "table",
labels = {'town': 'fish', 'analytics': 'yes'}
)
}}

select * from {{ ref('view_model') }}
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ def test__bigquery_simple_run(self):
self.run_dbt(['seed'])
self.run_dbt(['seed', '--full-refresh'])
results = self.run_dbt()
self.assertEqual(len(results), 6)
# Bump expected number of results when adding new model
self.assertEqual(len(results), 7)
self.assert_nondupes_pass()


Expand All @@ -63,7 +64,7 @@ class TestUnderscoreBigQueryRun(TestBaseBigQueryRun):
def test_bigquery_run_twice(self):
self.run_dbt(['seed'])
results = self.run_dbt()
self.assertEqual(len(results), 6)
self.assertEqual(len(results), 7)
results = self.run_dbt()
self.assertEqual(len(results), 6)
self.assertEqual(len(results), 7)
self.assert_nondupes_pass()