Merge pull request #2547 from azhard/bigquery-alter-column-type

Add support for altering BigQuery column types

CHANGELOG.md

@@ -9,7 +9,8 @@
 - Added intersection syntax for model selector ([#2167](https:/fishtown-analytics/dbt/issues/2167), [#2417](https:/fishtown-analytics/dbt/pull/2417))
 - Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https:/fishtown-analytics/dbt/issues/2052), [#2485](https:/fishtown-analytics/dbt/pull/2485))
 - Added support for renaming BigQuery relations ([#2520](https:/fishtown-analytics/dbt/issues/2520), [#2521](https:/fishtown-analytics/dbt/pull/2521))
-- Added support for BigQuery authorized views ([#1718](https:/fishtown-analytics/dbt/issues/1718), [#2517](https:/fishtown-analytics/dbt/issues/2517))
+- Added support for BigQuery authorized views ([#1718](https:/fishtown-analytics/dbt/issues/1718), [#2517](https:/fishtown-analytics/dbt/pull/2517))
+- Added support for altering BigQuery column types ([#2546](https:/fishtown-analytics/dbt/issues/2546), [#2547](https:/fishtown-analytics/dbt/pull/2547))
 
 ### Fixes
 - Fixed an error in create_adapter_plugins.py script when -dependency arg not passed ([#2507](https:/fishtown-analytics/dbt/issues/2507), [#2508](https:/fishtown-analytics/dbt/pull/2508))
@@ -20,7 +21,7 @@ Contributors:
  - [@alf-mindshift](https:/alf-mindshift) ([#2431](https:/fishtown-analytics/dbt/pull/2431))
  - [@scarrucciu](https:/scarrucciu) ([#2508](https:/fishtown-analytics/dbt/pull/2508))
  - [@southpolemonkey](https:/southpolemonkey) ([#2511](https:/fishtown-analytics/dbt/issues/2511))
- - [@azhard](https:/azhard) ([#2517](https:/fishtown-analytics/dbt/issues/2517), ([#2521](https:/fishtown-analytics/dbt/pull/2521)))
+ - [@azhard](https:/azhard) ([#2517](https:/fishtown-analytics/dbt/pull/2517), ([#2521](https:/fishtown-analytics/dbt/pull/2521)), [#2547](https:/fishtown-analytics/dbt/pull/2547))
 
 
 ## dbt 0.17.1 (Release TBD)

plugins/bigquery/dbt/include/bigquery/macros/adapters.sql

@@ -127,3 +127,31 @@
 {% macro bigquery__rename_relation(from_relation, to_relation) -%}
  {% do adapter.rename_relation(from_relation, to_relation) %}
 {% endmacro %}
+
+{% macro bigquery__alter_column_type(relation, column_name, new_column_type) -%}
+ {#
+ Changing a column's data type using a query requires you to scan the entire table.
+ The query charges can be significant if the table is very large.
+
+ https://cloud.google.com/bigquery/docs/manually-changing-schemas#changing_a_columns_data_type
+ #}
+ {% set relation_columns = get_columns_in_relation(relation) %}
+
+ {% set sql %}
+ select
+ {%- for col in relation_columns -%}
+ {% if col.column == column_name %}
+ CAST({{ col.quoted }} AS {{ new_column_type }}) AS {{ col.quoted }}
+ {%- else %}
+ {{ col.quoted }}
+ {%- endif %}
+ {%- if not loop.last %},{% endif -%}
+ {%- endfor %}
+ from {{ relation }}
+ {% endset %}
+
+ {% call statement('alter_column_type') %}
+ {{ create_table_as(False, relation, sql)}}
+ {%- endcall %}
+
+{% endmacro %}

test/integration/056_column_type_tests/bq_models_alter_type/altered_schema.yml

@@ -0,0 +1,10 @@
+version: 2
+models:
+ - name: model
+ tests:
+ - is_type:
+ column_map:
+ int64_col: ['string', 'not number']
+ float64_col: ['float', 'number']
+ numeric_col: ['numeric', 'number']
+ string_col: ['string', 'not number']

test/integration/056_column_type_tests/bq_models_alter_type/model.sql

@@ -0,0 +1,6 @@
+{{ config(materialized='table') }}
+select
+ CAST(1 as int64) as int64_col,
+ CAST(2.0 as float64) as float64_col,
+ CAST(3.0 as numeric) as numeric_col,
+ CAST('3' as string) as string_col,

test/integration/056_column_type_tests/macros/test_alter_column_type.sql

@@ -0,0 +1,5 @@
+-- Macro to alter a column type
+{% macro test_alter_column_type(model_name, column_name, new_column_type) %}
+ {% set relation = ref(model_name) %}
+ {{ alter_column_type(relation, column_name, new_column_type) }}
+{% endmacro %}

test/integration/056_column_type_tests/test_alter_column_types.py

@@ -0,0 +1,28 @@
+from test.integration.base import DBTIntegrationTest, use_profile
+import yaml
+
+
+class TestAlterColumnTypes(DBTIntegrationTest):
+ @property
+ def schema(self):
+ return '056_alter_column_types'
+
+ def run_and_alter_and_test(self, alter_column_type_args):
+ self.assertEqual(len(self.run_dbt(['run'])), 1)
+ self.run_dbt(['run-operation', 'test_alter_column_type', '--args', alter_column_type_args])
+ self.assertEqual(len(self.run_dbt(['test'])), 1)
+
+
+class TestBigQueryAlterColumnTypes(TestAlterColumnTypes):
+ @property
+ def models(self):
+ return 'bq_models_alter_type'
+
+ @use_profile('bigquery')
+ def test_bigquery_column_types(self):
+ alter_column_type_args = yaml.safe_dump({
+ 'model_name': 'model',
+ 'column_name': 'int64_col',
+ 'new_column_type': 'string'
+ })
+ self.run_and_alter_and_test(alter_column_type_args)