From ff31b277f65fee41c899aea6467fbd98f7e37b86 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 22 Sep 2020 13:45:59 -0600 Subject: [PATCH] add invocation_id to artifact metadata --- CHANGELOG.md | 2 +- core/dbt/contracts/util.py | 6 +++++- core/dbt/tracking.py | 7 +++++++ .../029_docs_generate_tests/test_docs_generate.py | 9 +++------ test/integration/042_sources_test/test_sources.py | 2 ++ test/unit/test_manifest.py | 10 +++++++++- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 559244ad62b..000c1191cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## dbt 0.19.0 (Release TBD) ### Breaking changes -- The format for sources.json, run-results.json, manifest.json, and catalog.json has changed to include a common metadata field ([#2761](https://github.com/fishtown-analytics/dbt/issues/2761), [#2778](https://github.com/fishtown-analytics/dbt/pull/2778)) +- The format for sources.json, run-results.json, manifest.json, and catalog.json has changed to include a common metadata field ([#2761](https://github.com/fishtown-analytics/dbt/issues/2761), [#2778](https://github.com/fishtown-analytics/dbt/pull/2778), [#2763](https://github.com/fishtown-analytics/dbt/issues/2763), [#2784](https://github.com/fishtown-analytics/dbt/pull/2784)) ### Features - dbt will compare configurations using the un-rendered form of the config block in dbt_project.yml ([#2713](https://github.com/fishtown-analytics/dbt/issues/2713), [#2735](https://github.com/fishtown-analytics/dbt/pull/2735)) diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index 827b971c344..7a1ff57f888 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -1,7 +1,7 @@ import dataclasses from datetime import datetime from typing import ( - List, Tuple, ClassVar, Type, TypeVar, Dict, Any + List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional ) from dbt.clients.system import write_json, read_json @@ -11,6 +11,7 @@ RuntimeException, ) from dbt.version import __version__ +from dbt.tracking import get_invocation_id from hologram import JsonSchemaMixin MacroKey = Tuple[str, str] @@ -129,6 +130,9 @@ class BaseArtifactMetadata(JsonSchemaMixin): generated_at: datetime = dataclasses.field( default_factory=datetime.utcnow ) + invocation_id: Optional[str] = dataclasses.field( + default_factory=get_invocation_id + ) def schema_version(name: str, version: int): diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index d9ea3ec0a91..127bcaabda2 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -421,6 +421,13 @@ def initialize_tracking(cookie_dir): active_user = User(None) +def get_invocation_id() -> Optional[str]: + if active_user is None: + return None + else: + return active_user.invocation_id + + class InvocationProcessor(logbook.Processor): def __init__(self): super().__init__() diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 5e5f93865ff..fc48bb5a8c8 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -5,6 +5,7 @@ import shutil import tempfile import time +import uuid from datetime import datetime from unittest.mock import ANY, patch @@ -12,6 +13,7 @@ from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ AnyString, AnyStringWith, normalize, Normalized +import dbt.tracking import dbt.version from dbt.exceptions import CompilationException @@ -508,8 +510,6 @@ def expected_postgres_references_catalog(self): }, } return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/catalog/v1.json', - 'dbt_version': dbt.version.__version__, 'nodes': { 'seed.test.seed': { 'unique_id': 'seed.test.seed', @@ -689,8 +689,6 @@ def expected_bigquery_complex_catalog(self): } return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/catalog/v1.json', - 'dbt_version': dbt.version.__version__, 'nodes': { 'model.test.clustered': { 'unique_id': 'model.test.clustered', @@ -776,8 +774,6 @@ def expected_redshift_incremental_catalog(self): my_schema_name = self.unique_schema() role = self.get_role() return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/catalog/v1.json', - 'dbt_version': dbt.version.__version__, 'nodes': { 'model.test.model': { 'unique_id': 'model.test.model', @@ -2566,6 +2562,7 @@ def verify_metadata(self, metadata, dbt_schema_version): assert metadata['dbt_version'] == dbt.version.__version__ assert 'dbt_schema_version' in metadata assert metadata['dbt_schema_version'] == dbt_schema_version + assert metadata['invocation_id'] == dbt.tracking.active_user.invocation_id def verify_manifest(self, expected_manifest): self.assertTrue(os.path.exists('./target/manifest.json')) diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index b7e6ebf7495..f273cf0f903 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -5,6 +5,7 @@ import yaml from dbt.exceptions import CompilationException +import dbt.tracking import dbt.version from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ AnyStringWith @@ -242,6 +243,7 @@ def _assert_freshness_results(self, path, state): self.freshness_start_time) assert data['metadata']['dbt_schema_version'] == 'https://schemas.getdbt.com/dbt/sources/v1.json' assert data['metadata']['dbt_version'] == dbt.version.__version__ + assert data['metadata']['invocation_id'] == dbt.tracking.active_user.invocation_id last_inserted_time = self.last_inserted_time diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 9ab1c35f9e4..de456823588 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -231,6 +231,7 @@ def test__no_nodes(self): 'generated_at': '2018-02-14T09:15:13Z', 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', 'dbt_version': dbt.version.__version__, + # invocation_id is None, so it will not be present }, 'docs': {}, 'disabled': [], @@ -322,6 +323,7 @@ def test__build_flat_graph(self): @mock.patch.object(tracking, 'active_user') def test_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' + mock_user.invocation_id = '01234567-0123-0123-0123-0123456789ab' mock_user.do_not_track = True now = datetime.utcnow() self.assertEqual( @@ -336,6 +338,7 @@ def test_metadata(self, mock_user): send_anonymous_usage_stats=False, adapter_type='postgres', generated_at=now, + invocation_id='01234567-0123-0123-0123-0123456789ab', ) ) @@ -343,6 +346,7 @@ def test_metadata(self, mock_user): @freezegun.freeze_time('2018-02-14T09:15:13Z') def test_no_nodes_with_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' + mock_user.invocation_id = '01234567-0123-0123-0123-0123456789ab' mock_user.do_not_track = True metadata = ManifestMetadata( project_id='098f6bcd4621d373cade4e832627b4f6', @@ -371,6 +375,7 @@ def test_no_nodes_with_metadata(self, mock_user): 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, 'adapter_type': 'postgres', + 'invocation_id': '01234567-0123-0123-0123-0123456789ab', }, 'disabled': [], } @@ -584,9 +589,10 @@ def setUp(self): @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__no_nodes(self): + metadata = ManifestMetadata(generated_at=datetime.utcnow(), invocation_id='01234567-0123-0123-0123-0123456789ab') manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, disabled=[], files={}, reports={}, - metadata=ManifestMetadata(generated_at=datetime.utcnow())) + metadata=metadata) self.assertEqual( manifest.writable_manifest().to_dict(), { @@ -600,6 +606,7 @@ def test__no_nodes(self): 'generated_at': '2018-02-14T09:15:13Z', 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v1.json', 'dbt_version': dbt.version.__version__, + 'invocation_id': '01234567-0123-0123-0123-0123456789ab', }, 'docs': {}, 'disabled': [], @@ -696,6 +703,7 @@ class TestManifestSearch(unittest.TestCase): _macros = [] _models = [] _docs = [] + @property def macros(self): return self._macros