Skip to content

Commit

Permalink
Use stats for date shards; always quote in catalog query
Browse files Browse the repository at this point in the history
  • Loading branch information
drewbanin committed Sep 29, 2019
1 parent 1a97915 commit 230c177
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 15 deletions.
1 change: 0 additions & 1 deletion core/dbt/task/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ class TableMetadata(JsonSchemaMixin):
name: str
comment: Optional[str]
owner: Optional[str]
shards: Optional[str]


@dataclass
Expand Down
25 changes: 12 additions & 13 deletions plugins/bigquery/dbt/adapters/bigquery/impl.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import copy

import dbt.deprecations
import dbt.exceptions
import dbt.flags as flags
Expand Down Expand Up @@ -393,19 +391,20 @@ def get_catalog(self, manifest):
Returns an agate.Table of catalog information.
"""

information_schemas = self._get_cache_schemas(manifest)
# information schemas are addressed from datasets on BigQuery
info_schemas = []
for info_schema_rel, schemas in information_schemas.items():
for schema in schemas:
real_info_schema = self.Relation.create(
database=info_schema_rel.database,
schema=schema
)
info_schemas.append(real_info_schema)
information_schemas = []
for database, schema in manifest.get_used_schemas():
information_schema = self.Relation.create(
database=database,
schema=schema,
quoting={
'database': True,
'schema': True
}
)
information_schemas.append(information_schema)

# make it a list so macros can index into it.
kwargs = {'information_schemas': info_schemas}
kwargs = {'information_schemas': information_schemas}
table = self.execute_macro(GET_CATALOG_MACRO_NAME,
kwargs=kwargs,
release=True)
Expand Down
16 changes: 15 additions & 1 deletion plugins/bigquery/dbt/include/bigquery/macros/catalog.sql
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,27 @@
else unsharded_tables.table_name
end as table_name,
unsharded_tables.table_type,
case when is_date_shard then table_shards else null end as table_shards,

columns.column_name,
columns.column_index,
columns.column_type,
columns.column_comment,

'# Date Shards' as `stats__date_shards__label`,
table_shards.shard_count as `stats__date_shards__value`,
'The number of date shards in this table' as `stats__date_shards__description`,
is_date_shard as `stats__date_shards__include`,

'Shard (min)' as `stats__date_shard_min__label`,
table_shards.shard_min as `stats__date_shard_min__value`,
'The first date shard in this table' as `stats__date_shard_min__description`,
is_date_shard as `stats__date_shard_min__include`,

'Shard (max)' as `stats__date_shard_max__label`,
table_shards.shard_max as `stats__date_shard_max__value`,
'The last date shard in this table' as `stats__date_shard_max__description`,
is_date_shard as `stats__date_shard_max__include`,

'Row Count' as `stats__row_count__label`,
row_count as `stats__row_count__value`,
'Approximate count of rows in this table' as `stats__row_count__description`,
Expand Down

0 comments on commit 230c177

Please sign in to comment.