From 3e40e71b96b299099afb40dc2b24d61d8edf5b32 Mon Sep 17 00:00:00 2001 From: Mark Scannell Date: Sat, 3 Oct 2020 13:01:25 +0100 Subject: [PATCH 1/5] Added dbt_invocation_id to BigQuery jobs --- plugins/bigquery/dbt/adapters/bigquery/connections.py | 8 +++++++- scripts/build-sdists.sh | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py index 5fb2857a043..cd50ca163d7 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py @@ -13,6 +13,7 @@ from dbt.utils import format_bytes, format_rows_number from dbt.clients import agate_helper, gcloud +from dbt.tracking import active_user from dbt.contracts.connection import ConnectionState from dbt.exceptions import ( FailedToConnectException, RuntimeException, DatabaseException @@ -249,7 +250,12 @@ def raw_execute(self, sql, fetch=False): logger.debug('On {}: {}', conn.name, sql) - job_params = {'use_legacy_sql': False} + job_params = { + 'use_legacy_sql': False, + 'labels': { + 'dbt_invocation_id': active_user.invocation_id, + }, + } priority = conn.credentials.priority if priority == Priority.Batch: diff --git a/scripts/build-sdists.sh b/scripts/build-sdists.sh index 8d0db621b23..91e9f70f5db 100755 --- a/scripts/build-sdists.sh +++ b/scripts/build-sdists.sh @@ -13,11 +13,11 @@ for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/s do rm -rf "$DBT_PATH"/"$SUBPATH"/dist cd "$DBT_PATH"/"$SUBPATH" - python setup.py sdist + python3 setup.py sdist cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/ done cd "$DBT_PATH" -python setup.py sdist +python3 setup.py sdist set +x From a6990c8fb8a68fd249f9c6da8012caf9d6f97323 Mon Sep 17 00:00:00 2001 From: Mark Scannell Date: Sat, 3 Oct 2020 13:04:56 +0100 Subject: [PATCH 2/5] fixe up --- scripts/build-sdists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build-sdists.sh b/scripts/build-sdists.sh index 91e9f70f5db..8d0db621b23 100755 --- a/scripts/build-sdists.sh +++ b/scripts/build-sdists.sh @@ -13,11 +13,11 @@ for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/s do rm -rf "$DBT_PATH"/"$SUBPATH"/dist cd "$DBT_PATH"/"$SUBPATH" - python3 setup.py sdist + python setup.py sdist cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/ done cd "$DBT_PATH" -python3 setup.py sdist +python setup.py sdist set +x From e29c14a22b892861bd11abaaeb87450aa2e3e451 Mon Sep 17 00:00:00 2001 From: Mark Scannell Date: Sat, 3 Oct 2020 13:09:04 +0100 Subject: [PATCH 3/5] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16cfa2050c1..546ec49c4cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Added schema and dbt versions to JSON artifacts ([#2670](https://github.com/fishtown-analytics/dbt/issues/2670), [#2767](https://github.com/fishtown-analytics/dbt/pull/2767)) - Added ability to snapshot hard-deleted records (opt-in with `invalidate_hard_deletes` config option). ([#249](https://github.com/fishtown-analytics/dbt/issues/249), [#2749](https://github.com/fishtown-analytics/dbt/pull/2749)) - Improved error messages for YAML selectors ([#2700](https://github.com/fishtown-analytics/dbt/issues/2700), [#2781](https://github.com/fishtown-analytics/dbt/pull/2781)) +- Added dbt_invocation_id for each BigQuery job to enable performance analysis ([#2483](https://github.com/fishtown-analytics/dbt/issues/2483)) ### Under the hood - Added strategy-specific validation to improve the relevancy of compilation errors for the `timestamp` and `check` snapshot strategies. (([#2787](https://github.com/fishtown-analytics/dbt/issues/2787), [#2791](https://github.com/fishtown-analytics/dbt/pull/2791)) From 26210216da61c6fd4e2c2a387c86d2112ab5f76a Mon Sep 17 00:00:00 2001 From: Mark Scannell Date: Sat, 3 Oct 2020 15:32:53 +0100 Subject: [PATCH 4/5] Only set invocation_id if tracking is enabled --- plugins/bigquery/dbt/adapters/bigquery/connections.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py index cd50ca163d7..c4b09eb2b48 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py @@ -252,11 +252,13 @@ def raw_execute(self, sql, fetch=False): job_params = { 'use_legacy_sql': False, - 'labels': { - 'dbt_invocation_id': active_user.invocation_id, - }, } + if active_user: + job_params['labels'] = { + 'dbt_invocation_id': active_user.invocation_id + } + priority = conn.credentials.priority if priority == Priority.Batch: job_params['priority'] = google.cloud.bigquery.QueryPriority.BATCH From 5d91aa3bcdd156156cfa950e83b379731f96a3bb Mon Sep 17 00:00:00 2001 From: Mark Scannell Date: Wed, 7 Oct 2020 10:55:35 +0100 Subject: [PATCH 5/5] Updated feature request resolution. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546ec49c4cf..dfa11e06568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Added schema and dbt versions to JSON artifacts ([#2670](https://github.com/fishtown-analytics/dbt/issues/2670), [#2767](https://github.com/fishtown-analytics/dbt/pull/2767)) - Added ability to snapshot hard-deleted records (opt-in with `invalidate_hard_deletes` config option). ([#249](https://github.com/fishtown-analytics/dbt/issues/249), [#2749](https://github.com/fishtown-analytics/dbt/pull/2749)) - Improved error messages for YAML selectors ([#2700](https://github.com/fishtown-analytics/dbt/issues/2700), [#2781](https://github.com/fishtown-analytics/dbt/pull/2781)) -- Added dbt_invocation_id for each BigQuery job to enable performance analysis ([#2483](https://github.com/fishtown-analytics/dbt/issues/2483)) +- Added dbt_invocation_id for each BigQuery job to enable performance analysis ([#2808](https://github.com/fishtown-analytics/dbt/issues/2808)) ### Under the hood - Added strategy-specific validation to improve the relevancy of compilation errors for the `timestamp` and `check` snapshot strategies. (([#2787](https://github.com/fishtown-analytics/dbt/issues/2787), [#2791](https://github.com/fishtown-analytics/dbt/pull/2791))