diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 7136b3e9027..c7fa6d79b7f 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -241,4 +241,4 @@ def check_schema_exists(self, database, schema, model_name=None): kwargs={'database': database, 'schema': schema}, connection_name=model_name ) - return results[0] > 0 + return results[0][0] > 0 diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql index 9b7c2634e0b..f363565a82d 100644 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ b/core/dbt/include/global_project/macros/adapters/common.sql @@ -200,7 +200,7 @@ {% macro check_schema_exists(database, schema) -%} - {{ return(adapter_macro('check_schema_exists', database)) }} + {{ return(adapter_macro('check_schema_exists', database, schema)) }} {% endmacro %} {% macro default__check_schema_exists(database, schema) -%} diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py index 66ffe9ede96..8fe5fbe4f00 100644 --- a/core/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -206,9 +206,11 @@ def do_skip(self, cause=None): self.skip_cause = cause @classmethod - def get_model_schemas(cls, manifest): + def get_model_schemas(cls, manifest, selected_uids): schemas = set() for node in manifest.nodes.values(): + if node.unique_id not in selected_uids: + continue if cls.is_refable(node) and not cls.is_ephemeral(node): schemas.add((node.database, node.schema)) @@ -219,7 +221,7 @@ def before_hooks(self, config, adapter, manifest): pass @classmethod - def before_run(self, config, adapter, manifest): + def before_run(self, config, adapter, manifest, selected_uids): pass @classmethod @@ -340,8 +342,8 @@ def safe_run_hooks(cls, config, adapter, manifest, hook_type, raise @classmethod - def create_schemas(cls, config, adapter, manifest): - required_schemas = cls.get_model_schemas(manifest) + def create_schemas(cls, config, adapter, manifest, selected_uids): + required_schemas = cls.get_model_schemas(manifest, selected_uids) # Snowflake needs to issue a "use {schema}" query, where schema # is the one defined in the profile. Create this schema if it @@ -365,10 +367,10 @@ def populate_adapter_cache(cls, config, adapter, manifest): adapter.set_relations_cache(manifest) @classmethod - def before_run(cls, config, adapter, manifest): + def before_run(cls, config, adapter, manifest, selected_uids): cls.populate_adapter_cache(config, adapter, manifest) cls.safe_run_hooks(config, adapter, manifest, RunHookType.Start, {}) - cls.create_schemas(config, adapter, manifest) + cls.create_schemas(config, adapter, manifest, selected_uids) @classmethod def print_results_line(cls, results, execution_time): diff --git a/core/dbt/runner.py b/core/dbt/runner.py index ed6defb8be0..d5b95b4da6f 100644 --- a/core/dbt/runner.py +++ b/core/dbt/runner.py @@ -171,8 +171,6 @@ def execute_nodes(self): dbt.ui.printer.print_timestamped_line(concurrency_line) dbt.ui.printer.print_timestamped_line("") - schemas = list(self.Runner.get_model_schemas(self.manifest)) - pool = ThreadPool(num_threads) try: self.run_queue(pool) @@ -303,10 +301,12 @@ def run(self): else: logger.info("") + selected_uids = frozenset(n.unique_id for n in self._flattened_nodes) try: self.Runner.before_hooks(self.config, adapter, self.manifest) started = time.time() - self.Runner.before_run(self.config, adapter, self.manifest) + self.Runner.before_run(self.config, adapter, self.manifest, + selected_uids) res = self.execute_nodes() self.Runner.after_run(self.config, adapter, res, self.manifest) elapsed = time.time() - started diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 82018a42286..8c426b9ff16 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -71,7 +71,10 @@ {% if database -%} {{ adapter.verify_database(database) }} {%- endif -%} - "select distinct nspname from pg_namespace" + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select distinct nspname from pg_namespace + {% endcall %} + {{ return(load_result('list_schemas').table) }} {% endmacro %} {% macro postgres__check_schema_exists(database, schema) -%} @@ -81,4 +84,5 @@ {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %} select count(*) from pg_namespace where nspname = '{{ schema }}' {% endcall %} + {{ return(load_result('check_schema_exists').table) }} {% endmacro %} diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql index 255e0a9df12..1324d70c19c 100644 --- a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql +++ b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql @@ -62,7 +62,7 @@ {% macro snowflake__check_schema_exists(database, schema) -%} {% call statement('check_schema_exists', fetch_result=True) -%} select count(*) - from {{ information_schema_name(database) }} + from {{ information_schema_name(database) }}.schemata where upper(schema_name) = upper('{{ schema }}') and upper(catalog_name) = upper('{{ database }}') {%- endcall %} diff --git a/test/integration/007_graph_selection_tests/models/never_selected.sql b/test/integration/007_graph_selection_tests/models/never_selected.sql new file mode 100644 index 00000000000..b1294f02014 --- /dev/null +++ b/test/integration/007_graph_selection_tests/models/never_selected.sql @@ -0,0 +1,5 @@ +{{ + config(schema='_and_then') +}} + +select * from {{ this.schema }}.seed diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index f4cad614ef9..5e4268462ba 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -11,6 +11,22 @@ def schema(self): def models(self): return "test/integration/007_graph_selection_tests/models" + def assert_correct_schemas(self): + exists = self.adapter.check_schema_exists( + self.default_database, + self.unique_schema(), + '__test' + ) + self.assertTrue(exists) + + schema = self.unique_schema()+'_and_then' + exists = self.adapter.check_schema_exists( + self.default_database, + schema, + '__test' + ) + self.assertFalse(exists) + @attr(type='postgres') def test__postgres__specific_model(self): self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") @@ -23,6 +39,7 @@ def test__postgres__specific_model(self): self.assertFalse('users_rollup' in created_models) self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) + self.assert_correct_schemas() @attr(type='postgres') def test__postgres__tags(self): @@ -36,6 +53,7 @@ def test__postgres__tags(self): self.assertFalse('emails' in created_models) self.assertTrue('users' in created_models) self.assertTrue('users_rollup' in created_models) + self.assert_correct_schemas() @attr(type='postgres') def test__postgres__tags_and_children(self): @@ -49,6 +67,7 @@ def test__postgres__tags_and_children(self): self.assertFalse('emails' in created_models) self.assertTrue('users_rollup' in created_models) self.assertTrue('users' in created_models) + self.assert_correct_schemas() @attr(type='snowflake') def test__snowflake__specific_model(self): @@ -62,7 +81,7 @@ def test__snowflake__specific_model(self): self.assertFalse('USERS_ROLLUP' in created_models) self.assertFalse('BASE_USERS' in created_models) self.assertFalse('EMAILS' in created_models) - + self.assert_correct_schemas() @attr(type='postgres') def test__postgres__specific_model_and_children(self): @@ -76,6 +95,7 @@ def test__postgres__specific_model_and_children(self): created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) + self.assert_correct_schemas() @attr(type='snowflake') def test__snowflake__specific_model_and_children(self): @@ -105,6 +125,7 @@ def test__postgres__specific_model_and_parents(self): created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) + self.assert_correct_schemas() @attr(type='snowflake') def test__snowflake__specific_model_and_parents(self): @@ -137,6 +158,7 @@ def test__postgres__specific_model_with_exclusion(self): self.assertFalse('base_users' in created_models) self.assertFalse('users_rollup' in created_models) self.assertFalse('emails' in created_models) + self.assert_correct_schemas() @attr(type='snowflake') def test__snowflake__specific_model_with_exclusion(self): @@ -164,3 +186,4 @@ def test__postgres__locally_qualified_name(self): self.assertNotIn('emails', created_models) self.assertIn('subdir', created_models) self.assertIn('nested_users', created_models) + self.assert_correct_schemas() diff --git a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py index c59dd6b524f..cd8cc191ef0 100644 --- a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py @@ -25,7 +25,7 @@ def packages_config(self): def run_schema_and_assert(self, include, exclude, expected_tests): self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(["deps"]) - results = self.run_dbt() + results = self.run_dbt(['run', '--exclude', 'never_selected']) self.assertEqual(len(results), 7) args = FakeArgs()