From b3a7b0e35b4183e4aca0d4bfec503c4db01eb113 Mon Sep 17 00:00:00 2001 From: Claire Carroll Date: Tue, 30 Mar 2021 10:45:26 -0400 Subject: [PATCH] Allow star macro to be case insensitive. Improve docs while we're here. (#348) Co-authored-by: Matthieu Di Mercurio --- CHANGELOG.md | 1 + README.md | 67 +++++++++++++++++-- integration_tests/models/sql/test_star.sql | 12 ++-- .../models/sql/test_star_aliases.sql | 4 +- macros/sql/star.sql | 18 +++-- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8807017c..28edb2c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Add new `accepted_range` test ([#276](https://github.com/fishtown-analytics/dbt-utils/pull/276) [@joellabes](https://github.com/joellabes)) * Make `expression_is_true` work as a column test (code originally in [#226](https://github.com/fishtown-analytics/dbt-utils/pull/226/) from [@elliottohara](https://github.com/elliottohara), merged via [#313]) * Allow individual columns in star macro to be aliased (code originally in [#230](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@elliottohara](https://github.com/elliottohara), merged via [#245]) +* Allow star macro to be case insensitive, and improve docs (code originally in [#281](https://github.com/fishtown-analytics/dbt-utils/pull/230/) via [@mdimercurio](https://github.com/mdimercurio), merged via [#348](https://github.com/fishtown-analytics/dbt-utils/pull/348/)) * Add new schema test, `not_accepted_values` ([#284](https://github.com/fishtown-analytics/dbt-utils/pull/284) [@JavierMonton](https://github.com/JavierMonton)) * Add new schema test, `fewer_rows_than` (code originally in [#221](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@dmarts](https://github.com/dmarts), merged via [#343]) diff --git a/README.md b/README.md index 1c2e4dc2..05414724 100644 --- a/README.md +++ b/README.md @@ -670,15 +670,74 @@ Usage: ``` #### star ([source](macros/sql/star.sql)) -This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro. This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias. -It also has an optional arg that allows aliasing of individual columns. +This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ ref('my_model') }}`, replacing star (`*`) with the star macro. + Usage: +```sql +select + {{ dbt_utils.star(ref('my_model')) }} +from {{ ref('my_model') }} +``` + +
Example compiled code + +```txt +select + id, + first_name, + deleted_at +from my_schema.my_model +``` + +

+ +```sql +select + {{ dbt_utils.star(ref('my_model'), except=['deleted_at']) }} +from {{ ref('my_model') }} as +``` + +
Example compiled code + +```txt +select + id, + first_name +from my_schema.my_model ``` + +

+ +```sql select -{{ dbt_utils.star(from=ref('my_model'), except=["exclude_field_1", "exclude_field_2"], aliases={"field_x":"pretty_name", "field_y":"another_pretty_name"}) }} -from {{ref('my_model')}} + {{ dbt_utils.star( + from=ref('my_model'), + relation_alias='my_alias', + except=["deleted_at", "exclude_field_2"], + case_sensitive_except=false, + aliases={"id":"customer_id"} + ) }} +from {{ ref('my_model') }} as my_alias ``` +
Example compiled code + +```txt +select + my_alias.id as customer_id, + my_alias.first_name +from my_schema.my_model as my_alias +``` + +

+ +Arguments: +- `from` (required): a [Relation](https://docs.getdbt.com/reference/dbt-classes#relation) (a `ref` or `source` function) that contains the list of columns you wish to select from +- `relation_alias` (optional, default=None): If used, the columns will be prefixed with this alias (i.e. `my_alias.my_column` instead of `my_column`.) Useful if your `from` statement aliases the relation (especially useful if you have a join in this query). +- `except` (optional, default=[]): a list of column names to exclude +- `case_sensitive_except` (optional, default=True): When true, columns in the `except` array must have the same casing as columns in the relation (i.e. `except=['my_col']` will _not_ exclude a column named 'MY_COL') +- `aliases` (optional, default={}): A dictionary of column aliases + #### union_relations ([source](macros/sql/union.sql)) This macro unions together an array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation), diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 3c1af078..a9a999b6 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,11 @@ - --- TODO : Should the star macro use a case-insensitive comparison for the `except` field on Snowflake? - -{% set exclude_field = 'FIELD_3' if target.type == 'snowflake' else 'field_3' %} - - with data as ( select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + {{ dbt_utils.star( + from=ref('data_star'), + except=['field_3'], + case_sensitive_except=False + ) }} from {{ ref('data_star') }} diff --git a/integration_tests/models/sql/test_star_aliases.sql b/integration_tests/models/sql/test_star_aliases.sql index 81bdb29f..9886fa50 100644 --- a/integration_tests/models/sql/test_star_aliases.sql +++ b/integration_tests/models/sql/test_star_aliases.sql @@ -1,7 +1,5 @@ --- TODO : Should the star macro use a case-insensitive comparison for the `except` field on Snowflake? - -{% set aliases = {'FIELD_3':'ALIASED'} if target.type == 'snowflake' else {'field_3':'aliased'} %} +{% set aliases = {'FIELD_3':'ALIASED'} if target.type == 'snowflake' else {'field_3':'aliased'} -%} with data as ( diff --git a/macros/sql/star.sql b/macros/sql/star.sql index f07599e9..5a0c0fa9 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,8 +1,8 @@ -{% macro star(from, relation_alias=False, except=[], aliases={}) -%} - {{ return(adapter.dispatch('star', packages = dbt_utils._get_utils_namespaces())(from, relation_alias, except, aliases)) }} +{% macro star(from, relation_alias=None, except=[], case_sensitive_except=True, aliases={}) -%} + {{ return(adapter.dispatch('star', packages = dbt_utils._get_utils_namespaces())(from, relation_alias, except, case_sensitive_except, aliases)) }} {% endmacro %} -{% macro default__star(from, relation_alias=False, except=[], aliases={}) -%} +{% macro default__star(from, relation_alias=None, except=[], case_sensitive_except=True, aliases={}) -%} {%- do dbt_utils._is_relation(from, 'star') -%} {%- do dbt_utils._is_ephemeral(from, 'star') -%} @@ -15,10 +15,14 @@ {%- set cols = adapter.get_columns_in_relation(from) -%} {%- for col in cols -%} - - {%- if col.column not in except -%} - {% do include_cols.append(col.column) %} - + {%- if case_sensitive_except %} + {%- if col.column not in except -%} + {% do include_cols.append(col.column) %} + {%- endif %} + {%- else %} + {%- if col.column|lower not in except|map('lower') -%} + {% do include_cols.append(col.column) %} + {%- endif %} {%- endif %} {%- endfor %}