From c74e9dde2b4a4d8bebca76916aca74b2f50c03eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 16:00:53 -0400 Subject: [PATCH] When parsing 'all_sources' should be a list of unique dirs (#5176) (#5256) * When parsing 'all_sources' should be a list of unique dirs * Changie * Fix some unit tests of all_source_paths * Convert 039_config_tests * Remove old 039_config_tests * Add test for duplicate directories in 'all_source_files' (cherry picked from commit f633e9936f62702a59fa661d261caaff9cefbe92) Co-authored-by: Gerda Shank --- .../unreleased/Fixes-20220427-102648.yaml | 7 + core/dbt/config/project.py | 6 +- core/dbt/tests/fixtures/project.py | 1 + .../039_config_tests/extra-alt/untagged.yml | 11 - .../039_config_tests/extra-alt/untagged2.yml | 11 - .../039_config_tests/models-alt/schema.yml | 31 -- .../models-alt/tagged/model.sql | 15 - .../039_config_tests/models-alt/untagged.sql | 5 - .../039_config_tests/models-get/any_model.sql | 2 - .../039_config_tests/models-ok/ok.sql | 1 - .../039_config_tests/models/schema.yml | 26 -- .../039_config_tests/models/tagged/model.sql | 15 - .../039_config_tests/models/untagged.sql | 5 - .../039_config_tests/project/dbt_project.yml | 8 - .../039_config_tests/project/profiles.yml | 24 -- .../039_config_tests/seeds-alt/some_seed.csv | 2 - .../039_config_tests/seeds/seed.csv | 2 - .../039_config_tests/test_configs.py | 350 ------------------ .../test_configs_in_schema_files.py | 115 ------ .../039_config_tests/tests/failing.sql | 2 - .../039_config_tests/tests/sleeper_agent.sql | 6 - test/unit/test_config.py | 6 +- tests/functional/configs/fixtures.py | 100 +++++ tests/functional/configs/test_configs.py | 65 ++++ .../configs/test_configs_in_schema_files.py | 215 +++++++++++ .../configs/test_disabled_configs.py | 92 +++++ tests/functional/configs/test_dupe_paths.py | 51 +++ tests/functional/configs/test_get_default.py | 27 ++ .../configs/test_get_project_config.py | 35 ++ tests/functional/configs/test_indiv_tests.py | 59 +++ .../functional/configs/test_unused_configs.py | 52 +++ 31 files changed, 712 insertions(+), 635 deletions(-) create mode 100644 .changes/unreleased/Fixes-20220427-102648.yaml delete mode 100644 test/integration/039_config_tests/extra-alt/untagged.yml delete mode 100644 test/integration/039_config_tests/extra-alt/untagged2.yml delete mode 100644 test/integration/039_config_tests/models-alt/schema.yml delete mode 100644 test/integration/039_config_tests/models-alt/tagged/model.sql delete mode 100644 test/integration/039_config_tests/models-alt/untagged.sql delete mode 100644 test/integration/039_config_tests/models-get/any_model.sql delete mode 100644 test/integration/039_config_tests/models-ok/ok.sql delete mode 100644 test/integration/039_config_tests/models/schema.yml delete mode 100644 test/integration/039_config_tests/models/tagged/model.sql delete mode 100644 test/integration/039_config_tests/models/untagged.sql delete mode 100644 test/integration/039_config_tests/project/dbt_project.yml delete mode 100644 test/integration/039_config_tests/project/profiles.yml delete mode 100644 test/integration/039_config_tests/seeds-alt/some_seed.csv delete mode 100644 test/integration/039_config_tests/seeds/seed.csv delete mode 100644 test/integration/039_config_tests/test_configs.py delete mode 100644 test/integration/039_config_tests/test_configs_in_schema_files.py delete mode 100644 test/integration/039_config_tests/tests/failing.sql delete mode 100644 test/integration/039_config_tests/tests/sleeper_agent.sql create mode 100644 tests/functional/configs/fixtures.py create mode 100644 tests/functional/configs/test_configs.py create mode 100644 tests/functional/configs/test_configs_in_schema_files.py create mode 100644 tests/functional/configs/test_disabled_configs.py create mode 100644 tests/functional/configs/test_dupe_paths.py create mode 100644 tests/functional/configs/test_get_default.py create mode 100644 tests/functional/configs/test_get_project_config.py create mode 100644 tests/functional/configs/test_indiv_tests.py create mode 100644 tests/functional/configs/test_unused_configs.py diff --git a/.changes/unreleased/Fixes-20220427-102648.yaml b/.changes/unreleased/Fixes-20220427-102648.yaml new file mode 100644 index 00000000000..62b5d93432a --- /dev/null +++ b/.changes/unreleased/Fixes-20220427-102648.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: When parsing 'all_sources' should be a list of unique dirs +time: 2022-04-27T10:26:48.648388-04:00 +custom: + Author: gshank + Issue: "5120" + PR: "5176" diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 825cc980fd5..db04b02bc53 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -132,7 +132,11 @@ def _all_source_paths( analysis_paths: List[str], macro_paths: List[str], ) -> List[str]: - return list(chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths)) + # We need to turn a list of lists into just a list, then convert to a set to + # get only unique elements, then back to a list + return list( + set(list(chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths))) + ) T = TypeVar("T") diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 8c9426c898c..2bb9169c2fd 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -182,6 +182,7 @@ def dbt_project_yml(project_root, project_config_update, logs_dir): if project_config_update: project_config.update(project_config_update) write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") + return project_config # Fixture to provide packages as either yaml or dictionary diff --git a/test/integration/039_config_tests/extra-alt/untagged.yml b/test/integration/039_config_tests/extra-alt/untagged.yml deleted file mode 100644 index 410c4057963..00000000000 --- a/test/integration/039_config_tests/extra-alt/untagged.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: untagged - description: "This is a model description" - meta: - owner: 'Somebody Else' - config: - meta: - owner: 'Julie Smith' - diff --git a/test/integration/039_config_tests/extra-alt/untagged2.yml b/test/integration/039_config_tests/extra-alt/untagged2.yml deleted file mode 100644 index 0e8cc6de7d0..00000000000 --- a/test/integration/039_config_tests/extra-alt/untagged2.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: untagged - description: "This is a model description" - tests: - - not_null: - error_if: ">2" - config: - error_if: ">2" - diff --git a/test/integration/039_config_tests/models-alt/schema.yml b/test/integration/039_config_tests/models-alt/schema.yml deleted file mode 100644 index d0937a6235a..00000000000 --- a/test/integration/039_config_tests/models-alt/schema.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 -sources: - - name: raw - database: "{{ target.database }}" - schema: "{{ target.schema }}" - tables: - - name: 'some_seed' - columns: - - name: id - -models: - - name: model - description: "This is a model description" - config: - tags: ['tag_in_schema'] - meta: - owner: 'Julie Smith' - my_attr: "{{ var('my_var') }}" - materialization: view - - columns: - - name: id - tests: - - not_null: - meta: - owner: 'Simple Simon' - - unique: - config: - meta: - owner: 'John Doe' - diff --git a/test/integration/039_config_tests/models-alt/tagged/model.sql b/test/integration/039_config_tests/models-alt/tagged/model.sql deleted file mode 100644 index 6dbc83c3d6e..00000000000 --- a/test/integration/039_config_tests/models-alt/tagged/model.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config( - materialized='view', - tags=['tag_1_in_model'], - ) -}} - -{{ - config( - materialized='table', - tags=['tag_2_in_model'], - ) -}} - -select 4 as id, 2 as value diff --git a/test/integration/039_config_tests/models-alt/untagged.sql b/test/integration/039_config_tests/models-alt/untagged.sql deleted file mode 100644 index a0dce3f086f..00000000000 --- a/test/integration/039_config_tests/models-alt/untagged.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ - config(materialized='table') -}} - -select id, value from {{ source('raw', 'some_seed') }} diff --git a/test/integration/039_config_tests/models-get/any_model.sql b/test/integration/039_config_tests/models-get/any_model.sql deleted file mode 100644 index a2743f38ca1..00000000000 --- a/test/integration/039_config_tests/models-get/any_model.sql +++ /dev/null @@ -1,2 +0,0 @@ --- models/any_model.sql -select {{ config.get('made_up_nonexistent_key', 'default_value') }} as col_value diff --git a/test/integration/039_config_tests/models-ok/ok.sql b/test/integration/039_config_tests/models-ok/ok.sql deleted file mode 100644 index b37cad1550b..00000000000 --- a/test/integration/039_config_tests/models-ok/ok.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('seed') }} \ No newline at end of file diff --git a/test/integration/039_config_tests/models/schema.yml b/test/integration/039_config_tests/models/schema.yml deleted file mode 100644 index 36b1054818b..00000000000 --- a/test/integration/039_config_tests/models/schema.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 -sources: - - name: raw - database: "{{ target.database }}" - schema: "{{ target.schema }}" - tables: - - name: 'seed' - identifier: "{{ var('seed_name', 'invalid') }}" - columns: - - name: id - tests: - - unique: - enabled: "{{ var('enabled_direct', None) | as_native }}" - - accepted_values: - enabled: "{{ var('enabled_direct', None) | as_native }}" - severity: "{{ var('severity_direct', None) | as_native }}" - values: [1,2] - -models: - - name: model - columns: - - name: id - tests: - - unique - - accepted_values: - values: [1,2,3,4] diff --git a/test/integration/039_config_tests/models/tagged/model.sql b/test/integration/039_config_tests/models/tagged/model.sql deleted file mode 100644 index 43aabe2102c..00000000000 --- a/test/integration/039_config_tests/models/tagged/model.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config( - materialized='view', - tags=['tag_two'], - ) -}} - -{{ - config( - materialized='table', - tags=['tag_three'], - ) -}} - -select 4 as id, 2 as value diff --git a/test/integration/039_config_tests/models/untagged.sql b/test/integration/039_config_tests/models/untagged.sql deleted file mode 100644 index 9f9bc85e111..00000000000 --- a/test/integration/039_config_tests/models/untagged.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ - config(materialized='table') -}} - -select id, value from {{ source('raw', 'seed') }} diff --git a/test/integration/039_config_tests/project/dbt_project.yml b/test/integration/039_config_tests/project/dbt_project.yml deleted file mode 100644 index b905a6bf33a..00000000000 --- a/test/integration/039_config_tests/project/dbt_project.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -config-version: 2 -model-paths: - - models-ok -name: test -profile: test -test-paths: [] -version: "1.0" \ No newline at end of file diff --git a/test/integration/039_config_tests/project/profiles.yml b/test/integration/039_config_tests/project/profiles.yml deleted file mode 100644 index 129ff1c3c0b..00000000000 --- a/test/integration/039_config_tests/project/profiles.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -config: - send_anonymous_usage_stats: false -test: - outputs: - default2: - dbname: dbt - host: localhost - pass: password - port: 5432 - schema: test16486474128741104874_config_039 - threads: 4 - type: postgres - user: root - noaccess: - dbname: dbt - host: localhost - pass: password - port: 5432 - schema: test16486474128741104874_config_039 - threads: 4 - type: postgres - user: noaccess - target: default2 \ No newline at end of file diff --git a/test/integration/039_config_tests/seeds-alt/some_seed.csv b/test/integration/039_config_tests/seeds-alt/some_seed.csv deleted file mode 100644 index 83f9676727c..00000000000 --- a/test/integration/039_config_tests/seeds-alt/some_seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,value -4,2 diff --git a/test/integration/039_config_tests/seeds/seed.csv b/test/integration/039_config_tests/seeds/seed.csv deleted file mode 100644 index 83f9676727c..00000000000 --- a/test/integration/039_config_tests/seeds/seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,value -4,2 diff --git a/test/integration/039_config_tests/test_configs.py b/test/integration/039_config_tests/test_configs.py deleted file mode 100644 index b8eb9d4c9f1..00000000000 --- a/test/integration/039_config_tests/test_configs.py +++ /dev/null @@ -1,350 +0,0 @@ -from datetime import datetime -import os -import shutil - -from test.integration.base import DBTIntegrationTest, use_profile -from dbt.exceptions import CompilationException -from dbt.config.utils import get_project_config -from argparse import Namespace - -class TestConfigs(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - def unique_schema(self): - return super().unique_schema().upper() - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'models': { - 'test': { - 'tagged': { - # the model configs will override this - 'materialized': 'invalid', - # the model configs will append to these - 'tags': ['tag_one'], - } - }, - }, - 'seeds': { - 'quote_columns': False, - }, - } - - @property - def models(self): - return "models" - - @use_profile('postgres') - def test_postgres_config_layering(self): - self.assertEqual(len(self.run_dbt(['seed'])), 1) - # test the project-level tag, and both config() call tags - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_one'])), 1) - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_two'])), 1) - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_three'])), 1) - self.assertTablesEqual('seed', 'model') - # make sure we overwrote the materialization properly - models = self.get_models_in_schema() - self.assertEqual(models['model'], 'table') - - -class TestTargetConfigs(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - def unique_schema(self): - return super().unique_schema().upper() - - @property - def models(self): - return "models" - - def setUp(self): - super().setUp() - self.init_targets = [d for d in os.listdir('.') if os.path.isdir(d) and d.startswith('target_')] - - def tearDown(self): - super().tearDown() - for d in self.new_dirs(): - shutil.rmtree(d) - - def new_dirs(self): - for d in os.listdir('.'): - if os.path.isdir(d) and d not in self.init_targets and d.startswith('target_'): - yield d - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'target-path': "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}", - 'seeds': { - 'quote_columns': False, - }, - } - - @use_profile('postgres') - def test_postgres_alternative_target_paths(self): - self.run_dbt(['seed']) - dirs = list(self.new_dirs()) - self.assertEqual(len(dirs), 1) - self.assertTrue(os.path.exists(os.path.join(dirs[0], 'manifest.json'))) - - -class TestDisabledConfigs(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - def postgres_profile(self): - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'default2': { - 'type': 'postgres', - # make sure you can do this and get an int out - 'threads': "{{ (1 + 3) | as_number }}", - 'host': self.database_host, - 'port': "{{ (5400 + 32) | as_number }}", - 'user': 'root', - 'pass': 'password', - 'dbname': 'dbt', - 'schema': self.unique_schema() - }, - 'disabled': { - 'type': 'postgres', - # make sure you can do this and get an int out - 'threads': "{{ (1 + 3) | as_number }}", - 'host': self.database_host, - 'port': "{{ (5400 + 32) | as_number }}", - 'user': 'root', - 'pass': 'password', - 'dbname': 'dbt', - 'schema': self.unique_schema() - }, - }, - 'target': 'default2' - } - } - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['tests'], - 'models': { - 'test': { - 'enabled': "{{ (target.name == 'default2' | as_bool) }}", - }, - }, - # set the `var` result in schema.yml to be 'seed', so that the - # `source` call can suceed. - 'vars': { - 'test': { - 'seed_name': 'seed', - } - }, - 'seeds': { - 'quote_columns': False, - 'test': { - 'seed': { - 'enabled': "{{ (target.name == 'default2') | as_bool }}", - }, - }, - }, - 'tests': { - 'test': { - 'enabled': "{{ (target.name == 'default2') | as_bool }}", - 'severity': 'WARN' - }, - }, - } - - @property - def models(self): - return "models" - - @use_profile('postgres') - def test_postgres_disable_seed_partial_parse(self): - self.run_dbt(['--partial-parse', 'seed', '--target', 'disabled']) - self.run_dbt(['--partial-parse', 'seed', '--target', 'disabled']) - - @use_profile('postgres') - def test_postgres_conditional_model(self): - # no seeds/models - enabled should eval to False because of the target - results = self.run_dbt(['seed', '--target', 'disabled']) - self.assertEqual(len(results), 0) - results = self.run_dbt(['run', '--target', 'disabled']) - self.assertEqual(len(results), 0) - results = self.run_dbt(['test', '--target', 'disabled']) - self.assertEqual(len(results), 0) - - # has seeds/models - enabled should eval to True because of the target - results = self.run_dbt(['seed']) - self.assertEqual(len(results), 1) - results = self.run_dbt(['run']) - self.assertEqual(len(results), 2) - results = self.run_dbt(['test']) - self.assertEqual(len(results), 5) - - -class TestUnusedModelConfigs(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['does-not-exist'], - 'models': { - 'test': { - 'enabled': True, - } - }, - 'seeds': { - 'quote_columns': False, - }, - 'sources': { - 'test': { - 'enabled': True, - } - }, - 'tests': { - 'test': { - 'enabled': True, - } - }, - } - - @property - def models(self): - return "empty-models" - - @use_profile('postgres') - def test_postgres_warn_unused_configuration_paths(self): - with self.assertRaises(CompilationException) as exc: - self.run_dbt(['--warn-error', 'seed']) - - self.assertIn('Configuration paths exist', str(exc.exception)) - self.assertIn('- sources.test', str(exc.exception)) - self.assertIn('- models.test', str(exc.exception)) - self.assertIn('- models.test', str(exc.exception)) - - self.run_dbt(['seed']) - - -class TestConfigIndivTests(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['tests'], - 'seeds': { - 'quote_columns': False, - }, - 'vars': { - 'test': { - 'seed_name': 'seed', - } - }, - 'tests': { - 'test': { - 'enabled': True, - 'severity': 'WARN' - } - } - } - - @use_profile('postgres') - def test_postgres_configuring_individual_tests(self): - self.assertEqual(len(self.run_dbt(['seed'])), 1) - self.assertEqual(len(self.run_dbt(['run'])), 2) - - # all tests on (minus sleeper_agent) + WARN - self.assertEqual(len(self.run_dbt(['test'])), 5) - - # turn off two of them directly - self.assertEqual(len(self.run_dbt(['test', '--vars', '{"enabled_direct": False}'])), 3) - - # turn on sleeper_agent data test directly - self.assertEqual(len(self.run_dbt(['test', '--models', 'sleeper_agent', - '--vars', '{"enabled_direct": True}'])), 1) - - # set three to ERROR directly - results = self.run_dbt(['test', '--models', 'config.severity:error', - '--vars', '{"enabled_direct": True, "severity_direct": "ERROR"}' - ], expect_pass = False) - self.assertEqual(len(results), 2) - self.assertEqual(results[0].status, 'fail') - self.assertEqual(results[1].status, 'fail') - - -class TestConfigGetDefault(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - @property - def models(self): - return "models-get" - - @use_profile('postgres') - def test_postgres_config_with_get_default(self): - # This test runs a model with a config.get(key, default) - # The default value is 'default_value' and causes an error - results = self.run_dbt(['run'], expect_pass=False) - self.assertEqual(len(results), 1) - self.assertEqual(str(results[0].status), 'error') - self.assertIn('column "default_value" does not exist', results[0].message) - - -class TestConfigUtils(DBTIntegrationTest): - @property - def schema(self): - return "config_039" - - @property - def models(self): - return "models" - - @property - def project_config(self): - project_path = os.path.dirname(os.path.realpath(__file__)) + "/project" - project_config = get_project_config( - project_path, "test", Namespace(profiles_dir=project_path) - ) - - # thanks yaml :( - project_config["name"] = str(project_config["name"]) - project_config["version"] = str(project_config["version"]) - return project_config - - @use_profile("postgres") - def test_postgres_get_project_config(self): - - seed_result = self.run_dbt(["seed"], expect_pass=True) - run_result = self.run_dbt(["run"], expect_pass=True) - self.assertEqual(len(seed_result), 1) - self.assertEqual(len(run_result), 1) diff --git a/test/integration/039_config_tests/test_configs_in_schema_files.py b/test/integration/039_config_tests/test_configs_in_schema_files.py deleted file mode 100644 index 7cf2df645f4..00000000000 --- a/test/integration/039_config_tests/test_configs_in_schema_files.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import shutil - -from test.integration.base import DBTIntegrationTest, use_profile, get_manifest, normalize -from dbt.exceptions import CompilationException, ParsingException - - -class TestSchemaFileConfigs(DBTIntegrationTest): - @property - def schema(self): - return "config_039-alt" - - def unique_schema(self): - return super().unique_schema().upper() - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-alt'], - 'models': { - '+meta': { - 'company': 'NuMade', - }, - 'test': { - '+meta': { - 'project': 'test', - }, - 'tagged': { - '+meta': { - 'team': 'Core Team', - }, - 'tags': ['tag_in_project'], - 'model': { - 'materialized': 'table', - '+meta': { - 'owner': 'Julie Dent', - }, - } - } - }, - }, - 'vars': { - 'test': { - 'my_var': 'TESTING', - } - }, - 'seeds': { - 'quote_columns': False, - }, - } - - @property - def models(self): - return "models-alt" - - @use_profile('postgres') - def test_postgres_config_layering(self): - self.assertEqual(len(self.run_dbt(['seed'])), 1) - # test the project-level tag, and both config() call tags - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_in_project'])), 1) - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_1_in_model'])), 1) - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_2_in_model'])), 1) - self.assertEqual(len(self.run_dbt(['run', '--model', 'tag:tag_in_schema'])), 1) - manifest = get_manifest() - model_id = 'model.test.model' - model_node = manifest.nodes[model_id] - meta_expected = {'company': 'NuMade', 'project': 'test', 'team': 'Core Team', 'owner': 'Julie Smith', 'my_attr': 'TESTING'} - self.assertEqual(model_node.meta, meta_expected) - self.assertEqual(model_node.config.meta, meta_expected) - model_tags = ['tag_1_in_model', 'tag_2_in_model', 'tag_in_project', 'tag_in_schema'] - model_node_tags = model_node.tags.copy() - model_node_tags.sort() - self.assertEqual(model_node_tags, model_tags) - model_node_config_tags = model_node.config.tags.copy() - model_node_config_tags.sort() - self.assertEqual(model_node_config_tags, model_tags) - model_meta = { - 'company': 'NuMade', - 'project': 'test', - 'team': 'Core Team', - 'owner': 'Julie Smith', - 'my_attr': "TESTING", - } - self.assertEqual(model_node.config.meta, model_meta) - # make sure we overwrote the materialization properly - models = self.get_models_in_schema() - self.assertEqual(models['model'], 'table') - self.assertTablesEqual('some_seed', 'model') - # look for test meta - schema_file_id = model_node.patch_path - schema_file = manifest.files[schema_file_id] - tests = schema_file.get_tests('models', 'model') - self.assertIn(tests[0], manifest.nodes) - test = manifest.nodes[tests[0]] - expected_meta = {'owner': 'Simple Simon'} - self.assertEqual(test.config.meta, expected_meta) - test = manifest.nodes[tests[1]] - expected_meta = {'owner': 'John Doe'} - self.assertEqual(test.config.meta, expected_meta) - - # copy a schema file with multiple metas - shutil.copyfile('extra-alt/untagged.yml', 'models-alt/untagged.yml') - with self.assertRaises(ParsingException): - results = self.run_dbt(["run"]) - - # copy a schema file with config key in top-level of test and in config dict - shutil.copyfile('extra-alt/untagged2.yml', 'models-alt/untagged.yml') - with self.assertRaises(CompilationException): - results = self.run_dbt(["run"]) - - def tearDown(self): - if os.path.exists(normalize('models-alt/untagged.yml')): - os.remove(normalize('models-alt/untagged.yml')) - diff --git a/test/integration/039_config_tests/tests/failing.sql b/test/integration/039_config_tests/tests/failing.sql deleted file mode 100644 index 3d18679164d..00000000000 --- a/test/integration/039_config_tests/tests/failing.sql +++ /dev/null @@ -1,2 +0,0 @@ - -select 1 as fun diff --git a/test/integration/039_config_tests/tests/sleeper_agent.sql b/test/integration/039_config_tests/tests/sleeper_agent.sql deleted file mode 100644 index 5c7d64a6e96..00000000000 --- a/test/integration/039_config_tests/tests/sleeper_agent.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config( - enabled = var('enabled_direct', False), - severity = var('severity_direct', 'WARN') -) }} - -select 1 as fun diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 48c713680e1..eb92c8d54e2 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -602,7 +602,7 @@ def test_defaults(self): self.assertEqual(project.seed_paths, ['seeds']) self.assertEqual(project.test_paths, ['tests']) self.assertEqual(project.analysis_paths, ['analyses']) - self.assertEqual(project.docs_paths, ['models', 'seeds', 'snapshots', 'analyses', 'macros']) + self.assertEqual(set(project.docs_paths), set(['models', 'seeds', 'snapshots', 'analyses', 'macros'])) self.assertEqual(project.asset_paths, []) self.assertEqual(project.target_path, 'target') self.assertEqual(project.clean_targets, ['target']) @@ -635,7 +635,7 @@ def test_implicit_overrides(self): 'target-path': 'other-target', }) project = project_from_config_norender(self.default_project_data) - self.assertEqual(project.docs_paths, ['other-models', 'seeds', 'snapshots', 'analyses', 'macros']) + self.assertEqual(set(project.docs_paths), set(['other-models', 'seeds', 'snapshots', 'analyses', 'macros'])) self.assertEqual(project.clean_targets, ['other-target']) def test_hashed_name(self): @@ -1215,7 +1215,7 @@ def test_from_args(self): self.assertEqual(config.seed_paths, ['seeds']) self.assertEqual(config.test_paths, ['tests']) self.assertEqual(config.analysis_paths, ['analyses']) - self.assertEqual(config.docs_paths, ['models', 'seeds', 'snapshots', 'analyses', 'macros']) + self.assertEqual(set(config.docs_paths), set(['models', 'seeds', 'snapshots', 'analyses', 'macros'])) self.assertEqual(config.asset_paths, []) self.assertEqual(config.target_path, 'target') self.assertEqual(config.clean_targets, ['target']) diff --git a/tests/functional/configs/fixtures.py b/tests/functional/configs/fixtures.py new file mode 100644 index 00000000000..eb744f93430 --- /dev/null +++ b/tests/functional/configs/fixtures.py @@ -0,0 +1,100 @@ +import pytest + +models__schema_yml = """ +version: 2 +sources: + - name: raw + database: "{{ target.database }}" + schema: "{{ target.schema }}" + tables: + - name: 'seed' + identifier: "{{ var('seed_name', 'invalid') }}" + columns: + - name: id + tests: + - unique: + enabled: "{{ var('enabled_direct', None) | as_native }}" + - accepted_values: + enabled: "{{ var('enabled_direct', None) | as_native }}" + severity: "{{ var('severity_direct', None) | as_native }}" + values: [1,2] + +models: + - name: model + columns: + - name: id + tests: + - unique + - accepted_values: + values: [1,2,3,4] + +""" + +models__untagged_sql = """ +{{ + config(materialized='table') +}} + +select id, value from {{ source('raw', 'seed') }} + +""" + +models__tagged__model_sql = """ +{{ + config( + materialized='view', + tags=['tag_two'], + ) +}} + +{{ + config( + materialized='table', + tags=['tag_three'], + ) +}} + +select 4 as id, 2 as value + +""" + +seeds__seed_csv = """id,value +4,2 +""" + +tests__failing_sql = """ + +select 1 as fun + +""" + +tests__sleeper_agent_sql = """ +{{ config( + enabled = var('enabled_direct', False), + severity = var('severity_direct', 'WARN') +) }} + +select 1 as fun + +""" + + +class BaseConfigProject: + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "untagged.sql": models__untagged_sql, + "tagged": {"model.sql": models__tagged__model_sql}, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seeds__seed_csv} + + @pytest.fixture(scope="class") + def tests(self): + return { + "failing.sql": tests__failing_sql, + "sleeper_agent.sql": tests__sleeper_agent_sql, + } diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py new file mode 100644 index 00000000000..b4b4cfa6f1b --- /dev/null +++ b/tests/functional/configs/test_configs.py @@ -0,0 +1,65 @@ +import pytest +import os + +from dbt.tests.util import run_dbt, check_relations_equal +from tests.functional.configs.fixtures import BaseConfigProject + + +class TestConfigs(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "tagged": { + # the model configs will override this + "materialized": "invalid", + # the model configs will append to these + "tags": ["tag_one"], + } + }, + }, + "seeds": { + "quote_columns": False, + }, + } + + def test_config_layering( + self, + project, + ): + # run seed + results = run_dbt(["seed"]) + assert len(results) == 1 + + # test the project-level tag, and both config() call tags + assert len(run_dbt(["run", "--model", "tag:tag_one"])) == 1 + assert len(run_dbt(["run", "--model", "tag:tag_two"])) == 1 + assert len(run_dbt(["run", "--model", "tag:tag_three"])) == 1 + check_relations_equal(project.adapter, ["seed", "model"]) + + # make sure we overwrote the materialization properly + tables = project.get_tables_in_schema() + assert tables["model"] == "table" + + +# In addition to testing an alternative target-paths setting, it tests that +# the attribute is jinja rendered and that the context "modules" works. +class TestTargetConfigs(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "target-path": "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}", + "seeds": { + "quote_columns": False, + }, + } + + def test_alternative_target_paths(self, project): + run_dbt(["seed"]) + + target_path = "" + for d in os.listdir("."): + if os.path.isdir(d) and d.startswith("target_"): + target_path = d + assert os.path.exists(os.path.join(project.project_root, target_path, "manifest.json")) diff --git a/tests/functional/configs/test_configs_in_schema_files.py b/tests/functional/configs/test_configs_in_schema_files.py new file mode 100644 index 00000000000..70c9c134d67 --- /dev/null +++ b/tests/functional/configs/test_configs_in_schema_files.py @@ -0,0 +1,215 @@ +import pytest + +from dbt.tests.util import run_dbt, get_manifest, check_relations_equal, write_file + +from dbt.exceptions import CompilationException, ParsingException + +models_alt__schema_yml = """ +version: 2 +sources: + - name: raw + database: "{{ target.database }}" + schema: "{{ target.schema }}" + tables: + - name: 'some_seed' + columns: + - name: id + +models: + - name: model + description: "This is a model description" + config: + tags: ['tag_in_schema'] + meta: + owner: 'Julie Smith' + my_attr: "{{ var('my_var') }}" + materialization: view + + columns: + - name: id + tests: + - not_null: + meta: + owner: 'Simple Simon' + - unique: + config: + meta: + owner: 'John Doe' +""" + +models_alt__untagged_sql = """ +{{ + config(materialized='table') +}} + +select id, value from {{ source('raw', 'some_seed') }} +""" + +models_alt__tagged__model_sql = """ +{{ + config( + materialized='view', + tags=['tag_1_in_model'], + ) +}} + +{{ + config( + materialized='table', + tags=['tag_2_in_model'], + ) +}} + +select 4 as id, 2 as value +""" + +seeds_alt__some_seed_csv = """id,value +4,2 +""" + +extra_alt__untagged_yml = """ +version: 2 + +models: + - name: untagged + description: "This is a model description" + meta: + owner: 'Somebody Else' + config: + meta: + owner: 'Julie Smith' +""" + +extra_alt__untagged2_yml = """ +version: 2 + +models: + - name: untagged + description: "This is a model description" + tests: + - not_null: + error_if: ">2" + config: + error_if: ">2" +""" + + +class TestSchemaFileConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models_alt__schema_yml, + "untagged.sql": models_alt__untagged_sql, + "tagged": {"model.sql": models_alt__tagged__model_sql}, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"some_seed.csv": seeds_alt__some_seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+meta": { + "company": "NuMade", + }, + "test": { + "+meta": { + "project": "test", + }, + "tagged": { + "+meta": { + "team": "Core Team", + }, + "tags": ["tag_in_project"], + "model": { + "materialized": "table", + "+meta": { + "owner": "Julie Dent", + }, + }, + }, + }, + }, + "vars": { + "test": { + "my_var": "TESTING", + } + }, + "seeds": { + "quote_columns": False, + }, + } + + def test_config_layering( + self, + project, + ): + + # run seed + assert len(run_dbt(["seed"])) == 1 + + # test the project-level tag, and both config() call tags + assert len(run_dbt(["run", "--model", "tag:tag_in_project"])) == 1 + assert len(run_dbt(["run", "--model", "tag:tag_1_in_model"])) == 1 + assert len(run_dbt(["run", "--model", "tag:tag_2_in_model"])) == 1 + assert len(run_dbt(["run", "--model", "tag:tag_in_schema"])) == 1 + + # Verify that model nodes have expected tags and meta + manifest = get_manifest(project.project_root) + model_id = "model.test.model" + model_node = manifest.nodes[model_id] + meta_expected = { + "company": "NuMade", + "project": "test", + "team": "Core Team", + "owner": "Julie Smith", + "my_attr": "TESTING", + } + assert model_node.meta == meta_expected + assert model_node.config.meta == meta_expected + model_tags = ["tag_1_in_model", "tag_2_in_model", "tag_in_project", "tag_in_schema"] + model_node_tags = model_node.tags.copy() + model_node_tags.sort() + assert model_node_tags == model_tags + model_node_config_tags = model_node.config.tags.copy() + model_node_config_tags.sort() + assert model_node_config_tags == model_tags + model_meta = { + "company": "NuMade", + "project": "test", + "team": "Core Team", + "owner": "Julie Smith", + "my_attr": "TESTING", + } + assert model_node.config.meta == model_meta + + # make sure we overwrote the materialization properly + tables = project.get_tables_in_schema() + assert tables["model"] == "table" + check_relations_equal(project.adapter, ["some_seed", "model"]) + + # look for test meta + schema_file_id = model_node.patch_path + schema_file = manifest.files[schema_file_id] + tests = schema_file.get_tests("models", "model") + assert tests[0] in manifest.nodes + test = manifest.nodes[tests[0]] + expected_meta = {"owner": "Simple Simon"} + assert test.config.meta == expected_meta + test = manifest.nodes[tests[1]] + expected_meta = {"owner": "John Doe"} + assert test.config.meta == expected_meta + + # copy a schema file with multiple metas + # shutil.copyfile('extra-alt/untagged.yml', 'models-alt/untagged.yml') + write_file(extra_alt__untagged_yml, project.project_root, "models", "untagged.yml") + with pytest.raises(ParsingException): + run_dbt(["run"]) + + # copy a schema file with config key in top-level of test and in config dict + # shutil.copyfile('extra-alt/untagged2.yml', 'models-alt/untagged.yml') + write_file(extra_alt__untagged2_yml, project.project_root, "models", "untagged.yml") + with pytest.raises(CompilationException): + run_dbt(["run"]) diff --git a/tests/functional/configs/test_disabled_configs.py b/tests/functional/configs/test_disabled_configs.py new file mode 100644 index 00000000000..ee56a39a867 --- /dev/null +++ b/tests/functional/configs/test_disabled_configs.py @@ -0,0 +1,92 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.configs.fixtures import BaseConfigProject + + +class TestDisabledConfigs(BaseConfigProject): + @pytest.fixture(scope="class") + def dbt_profile_data(self, unique_schema): + return { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": { + "type": "postgres", + # make sure you can do this and get an int out + "threads": "{{ (1 + 3) | as_number }}", + "host": "localhost", + "port": "{{ (5400 + 32) | as_number }}", + "user": "root", + "pass": "password", + "dbname": "dbt", + "schema": unique_schema, + }, + "disabled": { + "type": "postgres", + # make sure you can do this and get an int out + "threads": "{{ (1 + 3) | as_number }}", + "host": "localhost", + "port": "{{ (5400 + 32) | as_number }}", + "user": "root", + "pass": "password", + "dbname": "dbt", + "schema": unique_schema, + }, + }, + "target": "default", + }, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "enabled": "{{ (target.name == 'default' | as_bool) }}", + }, + }, + # set the `var` result in schema.yml to be 'seed', so that the + # `source` call can suceed. + "vars": { + "test": { + "seed_name": "seed", + } + }, + "seeds": { + "quote_columns": False, + "test": { + "seed": { + "enabled": "{{ (target.name == 'default') | as_bool }}", + }, + }, + }, + "tests": { + "test": { + "enabled": "{{ (target.name == 'default') | as_bool }}", + "severity": "WARN", + }, + }, + } + + def test_disable_seed_partial_parse(self, project): + run_dbt(["--partial-parse", "seed", "--target", "disabled"]) + run_dbt(["--partial-parse", "seed", "--target", "disabled"]) + + def test_conditional_model(self, project): + # no seeds/models - enabled should eval to False because of the target + results = run_dbt(["seed", "--target", "disabled"]) + assert len(results) == 0 + results = run_dbt(["run", "--target", "disabled"]) + assert len(results) == 0 + results = run_dbt(["test", "--target", "disabled"]) + assert len(results) == 0 + + # has seeds/models - enabled should eval to True because of the target + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 2 + results = run_dbt(["test"]) + assert len(results) == 5 diff --git a/tests/functional/configs/test_dupe_paths.py b/tests/functional/configs/test_dupe_paths.py new file mode 100644 index 00000000000..6170a971142 --- /dev/null +++ b/tests/functional/configs/test_dupe_paths.py @@ -0,0 +1,51 @@ +import pytest + +from dbt.tests.util import run_dbt + + +my_model_sql = """ +select 1 as fun +""" + +seed_csv = """id,value +4,2 +""" + +somedoc_md = """ +{% docs somedoc %} +Testing, testing +{% enddocs %} +""" + +schema_yml = """ +version: 2 +models: + - name: my_model + description: testing model +""" + + +# Either a docs or a yml file is necessary to see the problem +# when two of the paths in 'all_source_paths' are the same +class TestDupeProjectPaths: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "seed.csv": seed_csv, + "somedoc.md": somedoc_md, + "schema.yml": schema_yml, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "model-paths": ["models"], + "seed-paths": ["models"], + } + + def test_config_with_dupe_paths(self, project, dbt_project_yml): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 1 diff --git a/tests/functional/configs/test_get_default.py b/tests/functional/configs/test_get_default.py new file mode 100644 index 00000000000..597e88c6d65 --- /dev/null +++ b/tests/functional/configs/test_get_default.py @@ -0,0 +1,27 @@ +import pytest + +from dbt.tests.util import run_dbt + + +models_get__any_model_sql = """ +-- models/any_model.sql +select {{ config.get('made_up_nonexistent_key', 'default_value') }} as col_value + +""" + + +class TestConfigGetDefault: + @pytest.fixture(scope="class") + def models(self): + return {"any_model.sql": models_get__any_model_sql} + + def test_config_with_get_default( + self, + project, + ): + # This test runs a model with a config.get(key, default) + # The default value is 'default_value' and causes an error + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 1 + assert str(results[0].status) == "error" + assert 'column "default_value" does not exist' in results[0].message diff --git a/tests/functional/configs/test_get_project_config.py b/tests/functional/configs/test_get_project_config.py new file mode 100644 index 00000000000..9440d657cbe --- /dev/null +++ b/tests/functional/configs/test_get_project_config.py @@ -0,0 +1,35 @@ +import pytest +from argparse import Namespace + +from dbt.tests.util import run_dbt +from tests.functional.configs.fixtures import BaseConfigProject + +from dbt.config.utils import get_project_config + + +class TestConfigUtils(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + } + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": "select 1 as fun"} + + def test_get_project_config( + self, + project, + ): + result = run_dbt(["seed"], expect_pass=True) + assert len(result) == 1 + result = run_dbt(["run"], expect_pass=True) + assert len(result) == 1 + + project_config = get_project_config( + project.project_root, "test", Namespace(profiles_dir=project.profiles_dir) + ) + assert project_config diff --git a/tests/functional/configs/test_indiv_tests.py b/tests/functional/configs/test_indiv_tests.py new file mode 100644 index 00000000000..1da1652de27 --- /dev/null +++ b/tests/functional/configs/test_indiv_tests.py @@ -0,0 +1,59 @@ +import pytest + +from dbt.tests.util import run_dbt + +from tests.functional.configs.fixtures import BaseConfigProject + + +class TestConfigIndivTests(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + "vars": { + "test": { + "seed_name": "seed", + } + }, + "tests": {"test": {"enabled": True, "severity": "WARN"}}, + } + + def test_configuring_individual_tests( + self, + project, + ): + assert len(run_dbt(["seed"])) == 1 + assert len(run_dbt(["run"])) == 2 + + # all tests on (minus sleeper_agent) + WARN + assert len(run_dbt(["test"])) == 5 + + # turn off two of them directly + assert len(run_dbt(["test", "--vars", '{"enabled_direct": False}'])) == 3 + + # turn on sleeper_agent data test directly + assert ( + len( + run_dbt( + ["test", "--models", "sleeper_agent", "--vars", '{"enabled_direct": True}'] + ) + ) + == 1 + ) + + # set three to ERROR directly + results = run_dbt( + [ + "test", + "--models", + "config.severity:error", + "--vars", + '{"enabled_direct": True, "severity_direct": "ERROR"}', + ], + expect_pass=False, + ) + assert len(results) == 2 + assert results[0].status == "fail" + assert results[1].status == "fail" diff --git a/tests/functional/configs/test_unused_configs.py b/tests/functional/configs/test_unused_configs.py new file mode 100644 index 00000000000..7796472fea9 --- /dev/null +++ b/tests/functional/configs/test_unused_configs.py @@ -0,0 +1,52 @@ +import pytest + +from dbt.tests.util import run_dbt +from dbt.exceptions import CompilationException + +seeds__seed_csv = """id,value +4,2 +""" + + +class TestUnusedModelConfigs: + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seeds__seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "test-paths": ["does-not-exist"], + "models": { + "test": { + "enabled": True, + } + }, + "seeds": { + "quote_columns": False, + }, + "sources": { + "test": { + "enabled": True, + } + }, + "tests": { + "test": { + "enabled": True, + } + }, + } + + def test_warn_unused_configuration_paths( + self, + project, + ): + with pytest.raises(CompilationException) as excinfo: + run_dbt(["--warn-error", "seed"]) + + assert "Configuration paths exist" in str(excinfo.value) + assert "- sources.test" in str(excinfo.value) + assert "- models.test" in str(excinfo.value) + assert "- models.test" in str(excinfo.value) + + run_dbt(["seed"])