merged and updated PR for #1031

core/dbt/contracts/graph/parsed.py

@@ -47,6 +47,10 @@
  'materialized': {
  'type': 'string',
  },
+ 'persist_docs': {
+ 'type': 'object',
+ 'additionalProperties': True,
+ },
  'post-hook': {
  'type': 'array',
  'items': HOOK_CONTRACT,
@@ -86,8 +90,8 @@
  },
  'required': [
  'enabled', 'materialized', 'post-hook', 'pre-hook', 'vars',
- 'quoting', 'column_types', 'tags'
- ],
+ 'quoting', 'column_types', 'tags', 'persist_docs'
+ ]
 }
 
 

core/dbt/include/global_project/macros/etc/get_relation_comment.sql

@@ -0,0 +1,18 @@
+{% macro table_options() %}
+ {%- set raw_persist_docs = config.get('persist_docs', {}) -%}
+
+{%- endmacro -%}
+
+{% macro get_relation_comment(persist_docs, model) %}
+
+ {%- if persist_docs is not mapping -%}
+ {{ exceptions.raise_compiler_error("Invalid value provided for 'persist_docs'. Expected dict but got value: " ~ raw_persist_docs) }}
+ {% endif %}
+
+ {% if persist_docs.get('relation', false) %}
+ {{ return((model.description | tojson)[1:-1]) }}
+ {%- else -%}
+ {{ return(none) }}
+ {% endif %}
+
+{% endmacro %}

core/dbt/parser/source_config.py

@@ -7,7 +7,7 @@
 
 class SourceConfig(object):
  AppendListFields = {'pre-hook', 'post-hook', 'tags'}
- ExtendDictFields = {'vars', 'column_types', 'quoting'}
+ ExtendDictFields = {'vars', 'column_types', 'quoting', 'persist_docs'}
  ClobberFields = {
  'alias',
  'schema',
@@ -100,11 +100,12 @@ def update_in_model_config(self, config):
  self.in_model_config[key] = current
  elif key in self.ExtendDictFields:
  current = self.in_model_config.get(key, {})
- if not isinstance(current, dict):
+ try:
+ current.update(value)
+ except (ValueError, TypeError, AttributeError):
  dbt.exceptions.raise_compiler_error(
  'Invalid config field: "{}" must be a dict'.format(key)
  )
- current.update(value)
  self.in_model_config[key] = current
  else: # key in self.ClobberFields or self.AdapterSpecificConfigs
  self.in_model_config[key] = value

plugins/bigquery/dbt/include/bigquery/macros/adapters.sql

@@ -26,27 +26,43 @@
 
 {%- endmacro -%}
 
+{% macro bigquery_table_options(persist_docs, temporary) %}
+ {% set opts = {} %}
+
+ {% set description = get_relation_comment(persist_docs, model) %}
+ {%- if description is not none -%}
+ {% do opts.update({'description': "'" ~ description ~ "'"}) %}
+ {% endif %}
+ {% if temporary %}
+ {% do opts.update({'expiration_timestamp': 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'}) %}
+ {% endif %}
+
+ OPTIONS({% for opt_key, opt_val in opts.items() %}
+ {{ opt_key }}={{ opt_val }}
+ {% endfor %})
+{%- endmacro -%}
 
 {% macro bigquery__create_table_as(temporary, relation, sql) -%}
  {%- set raw_partition_by = config.get('partition_by', none) -%}
  {%- set raw_cluster_by = config.get('cluster_by', none) -%}
+ {%- set raw_persist_docs = config.get('persist_docs', {}) -%}
 
  create or replace table {{ relation }}
  {{ partition_by(raw_partition_by) }}
  {{ cluster_by(raw_cluster_by) }}
- {% if temporary %}
- OPTIONS(
- expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)
- )
- {% endif %}
+ {{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=temporary) }}
  as (
  {{ sql }}
  );
 {% endmacro %}
 
 
 {% macro bigquery__create_view_as(relation, sql) -%}
- create or replace view {{ relation }} as (
+ {%- set raw_persist_docs = config.get('persist_docs', {}) -%}
+
+ create or replace view {{ relation }}
+ {{ bigquery_table_options(persist_docs=raw_persist_docs, temporary=false) }}
+ as (
  {{ sql }}
  );
 {% endmacro %}

test/integration/001_simple_copy_test/models/advanced_incremental.sql

@@ -1,7 +1,8 @@
 {{
  config(
  materialized = "incremental",
- unique_key = "id"
+ unique_key = "id",
+ persist_docs = {"relation": true}
  )
 }}
 

test/integration/022_bigquery_test/models/schema.yml

@@ -1,6 +1,13 @@
 version: 2
 models:
 - name: view_model
+ description: |
+ View model description "with double quotes"
+ and with 'single quotes' as welll as other;
+ '''abc123'''
+ reserved -- characters
+ --
+ /* comment */
  columns:
  - name: dupe
  tests:
@@ -17,6 +24,13 @@ models:
  name: view_model
  type: view
 - name: table_model
+ description: |
+ View model description "with double quotes"
+ and with 'single quotes' as welll as other;
+ '''abc123'''
+ reserved -- characters
+ --
+ /* comment */
  columns:
  - name: id
  tests:

test/integration/022_bigquery_test/models/table_model.sql

@@ -1,4 +1,8 @@
-
-{{ config(materialized = "table") }}
+{{
+ config(
+ materialized = "table",
+ persist_docs={ "relation": true, "columns": true, "schema": true }
+ )
+}}
 
 select * from {{ ref('view_model') }}

test/integration/022_bigquery_test/models/view_model.sql

@@ -1,6 +1,7 @@
 {{
  config(
- materialized = "view"
+ materialized = "view",
+ persist_docs={ "relation": true, "columns": true, "schema": true }
  )
 }}