diff --git a/CHANGELOG.md b/CHANGELOG.md index d92f5232e62..1fe2f2d33a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features - Support optional `updated_at` config parameter with `check` strategy snapshots. If not supplied, will use current timestamp (default). ([#1844](https://github.com/fishtown-analytics/dbt/issues/1844), [#3376](https://github.com/fishtown-analytics/dbt/pull/3376)) - Add the opt-in `--use-experimental-parser` flag ([#3307](https://github.com/fishtown-analytics/dbt/issues/3307)) +- Store test failures in the database ([#517](https://github.com/fishtown-analytics/dbt/issues/517), [#903](https://github.com/fishtown-analytics/dbt/issues/903), [#2593](https://github.com/fishtown-analytics/dbt/issues/2593), [#3316](https://github.com/fishtown-analytics/dbt/issues/3316)) ### Fixes - Fix compiled sql for ephemeral models ([#3317](https://github.com/fishtown-analytics/dbt/issues/3317), [#3318](https://github.com/fishtown-analytics/dbt/pull/3318)) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 8687c808920..5e0fa259f2a 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -182,8 +182,7 @@ def add_ephemeral_prefix(self, name: str): def _get_relation_name(self, node: ParsedNode): relation_name = None - if (node.resource_type in NodeType.refable() and - not node.is_ephemeral_model): + if node.is_relational and not node.is_ephemeral_model: adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index 8a4361b655f..3f78963649e 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -120,10 +120,14 @@ class CompiledSchemaTestNode(CompiledNode, HasTestMetadata): config: TestConfig = field(default_factory=TestConfig) def same_config(self, other) -> bool: - return ( - self.unrendered_config.get('severity') == - other.unrendered_config.get('severity') - ) + comparisons = [ + self.unrendered_config.get(modifier) == other.unrendered_config.get(modifier) or ( + self.unrendered_config.get(modifier) is None and + other.unrendered_config.get(modifier) is None + ) + for modifier in ('severity', 'store_failures') + ] + return all(comparisons) def same_column_name(self, other) -> bool: return self.column_name == other.column_name diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index e3566e2fff7..180b382689d 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -436,8 +436,13 @@ class SeedConfig(NodeConfig): @dataclass class TestConfig(NodeConfig): + schema: Optional[str] = field( + default='dbt_test__audit', + metadata=CompareBehavior.Exclude.meta(), + ) materialized: str = 'test' severity: Severity = Severity('ERROR') + store_failures: Optional[bool] = None @dataclass diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index c6d8cd9fd1a..e6b4a851a17 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -115,6 +115,21 @@ class ParsedNodeMixins(dbtClassMixin): def is_refable(self): return self.resource_type in NodeType.refable() + @property + def should_store_failures(self): + return self.resource_type == NodeType.Test and ( + self.config.store_failures if self.config.store_failures is not None + else flags.STORE_FAILURES + ) + + # will this node map to an object in the database? + @property + def is_relational(self): + return ( + self.resource_type in NodeType.refable() or + self.should_store_failures + ) + @property def is_ephemeral(self): return self.config.materialized == 'ephemeral' @@ -370,10 +385,14 @@ class ParsedSchemaTestNode(ParsedNode, HasTestMetadata): config: TestConfig = field(default_factory=TestConfig) def same_config(self, other) -> bool: - return ( - self.unrendered_config.get('severity') == - other.unrendered_config.get('severity') - ) + comparisons = [ + self.unrendered_config.get(modifier) == other.unrendered_config.get(modifier) or ( + self.unrendered_config.get(modifier) is None and + other.unrendered_config.get(modifier) is None + ) + for modifier in ('severity', 'store_failures') + ] + return all(comparisons) def same_column_name(self, other) -> bool: return self.column_name == other.column_name diff --git a/core/dbt/flags.py b/core/dbt/flags.py index cc721bd6da7..c37f3e465cd 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -17,6 +17,7 @@ WRITE_JSON = None PARTIAL_PARSE = None USE_COLORS = None +STORE_FAILURES = None def env_set_truthy(key: str) -> Optional[str]: @@ -54,7 +55,8 @@ def _get_context(): def reset(): global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \ - USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS + USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \ + STORE_FAILURES STRICT_MODE = False FULL_REFRESH = False @@ -66,11 +68,13 @@ def reset(): PARTIAL_PARSE = False MP_CONTEXT = _get_context() USE_COLORS = True + STORE_FAILURES = False def set_from_args(args): global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \ - USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS + USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \ + STORE_FAILURES USE_CACHE = getattr(args, 'use_cache', USE_CACHE) @@ -94,6 +98,8 @@ def set_from_args(args): if use_colors_override is not None: USE_COLORS = use_colors_override + STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES) + # initialize everything to the defaults on module load reset() diff --git a/core/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql index c83ce4c8d1f..7010f384112 100644 --- a/core/dbt/include/global_project/macros/materializations/helpers.sql +++ b/core/dbt/include/global_project/macros/materializations/helpers.sql @@ -72,3 +72,12 @@ {% endif %} {% do return(config_full_refresh) %} {% endmacro %} + + +{% macro should_store_failures() %} + {% set config_store_failures = config.get('store_failures') %} + {% if config_store_failures is none %} + {% set config_store_failures = flags.STORE_FAILURES %} + {% endif %} + {% do return(config_store_failures) %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/test.sql b/core/dbt/include/global_project/macros/materializations/test.sql index e832ab0e89c..98040cc0049 100644 --- a/core/dbt/include/global_project/macros/materializations/test.sql +++ b/core/dbt/include/global_project/macros/materializations/test.sql @@ -1,10 +1,46 @@ {%- materialization test, default -%} + {% set relations = [] %} + + {% if should_store_failures() %} + + {% set identifier = model['alias'] %} + {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + {% set target_relation = api.Relation.create( + identifier=identifier, schema=schema, database=database, type='table') -%} %} + + {% if old_relation %} + {% do adapter.drop_relation(old_relation) %} + {% endif %} + + {% call statement(auto_begin=True) %} + {{ create_table_as(False, target_relation, sql) }} + {% endcall %} + + {% do relations.append(target_relation) %} + + {% set main_sql %} + select count(*) as validation_errors + from {{ target_relation }} + {% endset %} + + {{ adapter.commit() }} + + {% else %} + + {% set main_sql %} + select count(*) as validation_errors + from ( + {{ sql }} + ) _dbt_internal_test + {% endset %} + + {% endif %} + {% call statement('main', fetch_result=True) -%} - select count(*) as validation_errors - from ( - {{ sql }} - ) _dbt_internal_test + {{ main_sql }} {%- endcall %} + + {{ return({'relations': relations}) }} {%- endmaterialization -%} diff --git a/core/dbt/include/global_project/macros/schema_tests/accepted_values.sql b/core/dbt/include/global_project/macros/schema_tests/accepted_values.sql index 21f85064b28..e9c5a309c85 100644 --- a/core/dbt/include/global_project/macros/schema_tests/accepted_values.sql +++ b/core/dbt/include/global_project/macros/schema_tests/accepted_values.sql @@ -2,16 +2,16 @@ with all_values as ( - select distinct - {{ column_name }} as value_field + select + {{ column_name }} as value_field, + count(*) as n_records from {{ model }} + group by 1 ) -select - value_field - +select * from all_values where value_field not in ( {% for value in values -%} diff --git a/core/dbt/main.py b/core/dbt/main.py index 8be10ce7207..ae53b13af46 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -719,6 +719,13 @@ def _build_test_subparser(subparsers, base_subparser): Stop execution upon a first test failure. ''' ) + sub.add_argument( + '--store-failures', + action='store_true', + help=''' + Store test results (failing rows) in the database + ''' + ) sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test') return sub diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 04eb6619b1b..b9790d77da0 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -655,7 +655,7 @@ def _check_resource_uniqueness( alias_resources: Dict[str, ManifestNode] = {} for resource, node in manifest.nodes.items(): - if node.resource_type not in NodeType.refable(): + if not node.is_relational: continue # appease mypy - sources aren't refable! assert not isinstance(node, ParsedSourceDefinition) diff --git a/core/dbt/parser/schema_test_builders.py b/core/dbt/parser/schema_test_builders.py index a7a14f5c694..a6c312990d0 100644 --- a/core/dbt/parser/schema_test_builders.py +++ b/core/dbt/parser/schema_test_builders.py @@ -41,16 +41,21 @@ def get_nice_schema_test_name( clean_flat_args = [re.sub('[^0-9a-zA-Z_]+', '_', arg) for arg in flat_args] unique = "__".join(clean_flat_args) - cutoff = 32 - if len(unique) <= cutoff: - label = unique - else: - label = hashlib.md5(unique.encode('utf-8')).hexdigest() + # for the file path + alias, the name must be <64 characters + # if the full name is too long, include the first 30 identifying chars plus + # a 32-character hash of the full contents + + test_identifier = '{}_{}'.format(test_type, test_name) + full_name = '{}_{}'.format(test_identifier, unique) - filename = '{}_{}_{}'.format(test_type, test_name, label) - name = '{}_{}_{}'.format(test_type, test_name, unique) + if len(full_name) >= 64: + test_trunc_identifier = test_identifier[:30] + label = hashlib.md5(full_name.encode('utf-8')).hexdigest() + short_name = '{}_{}'.format(test_trunc_identifier, label) + else: + short_name = full_name - return filename, name + return short_name, full_name @dataclass @@ -185,7 +190,7 @@ class TestBuilder(Generic[Testable]): r'(?P([a-zA-Z_][0-9a-zA-Z_]*))' ) # kwargs representing test configs - MODIFIER_ARGS = ('severity', 'tags', 'enabled') + MODIFIER_ARGS = ('severity', 'tags', 'enabled', 'store_failures') def __init__( self, @@ -231,6 +236,10 @@ def __init__( self.compiled_name: str = compiled_name self.fqn_name: str = fqn_name + # use hashed name as alias if too long + if compiled_name != fqn_name: + self.modifiers['alias'] = compiled_name + def _bad_type(self) -> TypeError: return TypeError('invalid target type "{}"'.format(type(self.target))) @@ -271,6 +280,9 @@ def extract_test_args(test, name=None) -> Tuple[str, Dict[str, Any]]: def enabled(self) -> Optional[bool]: return self.modifiers.get('enabled') + def alias(self) -> Optional[str]: + return self.modifiers.get('alias') + def severity(self) -> Optional[str]: sev = self.modifiers.get('severity') if sev: @@ -278,6 +290,9 @@ def severity(self) -> Optional[str]: else: return None + def store_failures(self) -> Optional[bool]: + return self.modifiers.get('store_failures') + def tags(self) -> List[str]: tags = self.modifiers.get('tags', []) if isinstance(tags, str): diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 8a04067a344..a0d0f9d0e07 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -493,6 +493,11 @@ def render_test_update(self, node, config, builder): node.config['severity'] = builder.severity() if builder.enabled() is not None: node.config['enabled'] = builder.enabled() + # note: this does not respect generate_alias_name() macro + if builder.alias() is not None: + node.unrendered_config['alias'] = builder.alias() + node.config['alias'] = builder.alias() + node.alias = builder.alias() # source node tests are processed at patch_source time if isinstance(builder.target, UnpatchedSourceDefinition): sources = [builder.target.fqn[-2], builder.target.fqn[-1]] diff --git a/core/dbt/task/printer.py b/core/dbt/task/printer.py index 0054aa22661..6ac3bc0bc1f 100644 --- a/core/dbt/task/printer.py +++ b/core/dbt/task/printer.py @@ -306,6 +306,13 @@ def print_run_result_error( logger.info(" compiled SQL at {}".format( result.node.compiled_path)) + if result.node.should_store_failures: + with TextOnly(): + logger.info("") + msg = f"select * from {result.node.relation_name}" + border = '-' * len(msg) + logger.info(f" See test failures:\n {border}\n {msg}\n {border}") + elif result.message is not None: first = True for line in result.message.split("\n"): diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 98af7546c87..ed9bab6e2f7 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -455,7 +455,7 @@ def get_model_schemas( for node in self.manifest.nodes.values(): if node.unique_id not in selected_uids: continue - if node.is_refable and not node.is_ephemeral: + if node.is_relational and not node.is_ephemeral: relation = adapter.Relation.create_from(self.config, node) result.add(relation.without_identifier()) @@ -525,7 +525,6 @@ def create_schema(relation: BaseRelation) -> None: db_schema = (db_lower, schema.lower()) if db_schema not in existing_schemas_lowered: existing_schemas_lowered.add(db_schema) - fut = tpe.submit_connected( adapter, f'create_{info.database or ""}_{info.schema}', create_schema, info diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 4967f413d3d..98a2d55bb2b 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -986,9 +986,10 @@ def rendered_tst_config(self, **updates): 'vars': {}, 'tags': [], 'severity': 'ERROR', + 'store_failures': None, 'full_refresh': None, 'database': None, - 'schema': None, + 'schema': 'dbt_test__audit', 'alias': None, } result.update(updates) @@ -1054,6 +1055,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): snapshot_path = self.dir(os.path.join('snapshot', 'snapshot_seed.sql')) my_schema_name = self.unique_schema() + test_audit_schema = my_schema_name + '_dbt_test__audit' if model_database is None: model_database = self.alternative_database @@ -1350,7 +1352,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, - 'schema': my_schema_name, + 'schema': test_audit_schema, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, @@ -1439,7 +1441,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, - 'schema': my_schema_name, + 'schema': test_audit_schema, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, @@ -1484,7 +1486,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): 'relation_name': None, 'resource_type': 'test', 'root_path': self.test_root_realpath, - 'schema': my_schema_name, + 'schema': test_audit_schema, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, diff --git a/test/integration/047_dbt_ls_test/test_ls.py b/test/integration/047_dbt_ls_test/test_ls.py index 747c4a1f947..38bc95df6fc 100644 --- a/test/integration/047_dbt_ls_test/test_ls.py +++ b/test/integration/047_dbt_ls_test/test_ls.py @@ -336,6 +336,7 @@ def expect_test_output(self): 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', + 'store_failures': None, 'tags': [], 'pre-hook': [], 'quoting': {}, @@ -344,7 +345,7 @@ def expect_test_output(self): 'persist_docs': {}, 'full_refresh': None, 'database': None, - 'schema': None, + 'schema': 'dbt_test__audit', 'alias': None, }, 'alias': 'not_null_outer_id', @@ -360,6 +361,7 @@ def expect_test_output(self): 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', + 'store_failures': None, 'tags': [], 'pre-hook': [], 'quoting': {}, @@ -368,7 +370,7 @@ def expect_test_output(self): 'persist_docs': {}, 'full_refresh': None, 'database': None, - 'schema': None, + 'schema': 'dbt_test__audit', 'alias': None, }, 'alias': 't', @@ -384,6 +386,7 @@ def expect_test_output(self): 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', + 'store_failures': None, 'tags': [], 'pre-hook': [], 'quoting': {}, @@ -392,7 +395,7 @@ def expect_test_output(self): 'persist_docs': {}, 'full_refresh': None, 'database': None, - 'schema': None, + 'schema': 'dbt_test__audit', 'alias': None, }, 'alias': 'unique_outer_id', diff --git a/test/integration/066_test_selection_tests/test_selection_expansion.py b/test/integration/066_test_selection_tests/test_selection_expansion.py index 13ab812298a..6a7401d926f 100644 --- a/test/integration/066_test_selection_tests/test_selection_expansion.py +++ b/test/integration/066_test_selection_tests/test_selection_expansion.py @@ -7,7 +7,7 @@ class TestSelectionExpansion(DBTIntegrationTest): @property def schema(self): - return "test_selection_expansion_065" + return "test_selection_expansion_066" @property def models(self): diff --git a/test/integration/067_store_test_failures_tests/data/expected/expected_accepted_values.csv b/test/integration/067_store_test_failures_tests/data/expected/expected_accepted_values.csv new file mode 100644 index 00000000000..02f28435b46 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/data/expected/expected_accepted_values.csv @@ -0,0 +1,3 @@ +value_field,n_records +Gary,1 +Rose,1 diff --git a/test/integration/067_store_test_failures_tests/data/expected/expected_failing_test.csv b/test/integration/067_store_test_failures_tests/data/expected/expected_failing_test.csv new file mode 100644 index 00000000000..d9e7257f122 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/data/expected/expected_failing_test.csv @@ -0,0 +1,11 @@ +id,first_name,last_name,email,gender,ip_address +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 diff --git a/test/integration/067_store_test_failures_tests/data/expected/expected_not_null_problematic_model_id.csv b/test/integration/067_store_test_failures_tests/data/expected/expected_not_null_problematic_model_id.csv new file mode 100644 index 00000000000..95fef8a2594 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/data/expected/expected_not_null_problematic_model_id.csv @@ -0,0 +1,3 @@ +id,first_name,last_name,email,gender,ip_address +,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 diff --git a/test/integration/067_store_test_failures_tests/data/expected/expected_unique_problematic_model_id.csv b/test/integration/067_store_test_failures_tests/data/expected/expected_unique_problematic_model_id.csv new file mode 100644 index 00000000000..17841dfa3d9 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/data/expected/expected_unique_problematic_model_id.csv @@ -0,0 +1,3 @@ +id,n_records +2,2 +1,2 \ No newline at end of file diff --git a/test/integration/067_store_test_failures_tests/data/people.csv b/test/integration/067_store_test_failures_tests/data/people.csv new file mode 100644 index 00000000000..d9e7257f122 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/data/people.csv @@ -0,0 +1,11 @@ +id,first_name,last_name,email,gender,ip_address +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 diff --git a/test/integration/067_store_test_failures_tests/models/fine_model.sql b/test/integration/067_store_test_failures_tests/models/fine_model.sql new file mode 100644 index 00000000000..94b923a17c2 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/models/fine_model.sql @@ -0,0 +1 @@ +select * from {{ ref('people') }} diff --git a/test/integration/067_store_test_failures_tests/models/fine_model_but_with_a_no_good_very_long_name.sql b/test/integration/067_store_test_failures_tests/models/fine_model_but_with_a_no_good_very_long_name.sql new file mode 100644 index 00000000000..97536ffaf06 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/models/fine_model_but_with_a_no_good_very_long_name.sql @@ -0,0 +1 @@ +select 1 as quite_long_column_name diff --git a/test/integration/067_store_test_failures_tests/models/problematic_model.sql b/test/integration/067_store_test_failures_tests/models/problematic_model.sql new file mode 100644 index 00000000000..e780d6b001e --- /dev/null +++ b/test/integration/067_store_test_failures_tests/models/problematic_model.sql @@ -0,0 +1,11 @@ +select * from {{ ref('people') }} + +union all + +select * from {{ ref('people') }} +where id in (1,2) + +union all + +select null as id, first_name, last_name, email, gender, ip_address from {{ ref('people') }} +where id in (3,4) diff --git a/test/integration/067_store_test_failures_tests/models/schema.yml b/test/integration/067_store_test_failures_tests/models/schema.yml new file mode 100644 index 00000000000..75a3a380afc --- /dev/null +++ b/test/integration/067_store_test_failures_tests/models/schema.yml @@ -0,0 +1,39 @@ +version: 2 + +models: + + - name: fine_model + columns: + - name: id + tests: + - unique + - not_null + + - name: problematic_model + columns: + - name: id + tests: + - unique + - not_null + - name: first_name + tests: + # test truncation of really long test name + - accepted_values: + values: + - Jack + - Kathryn + - Gerald + - Bonnie + - Harold + - Jacqueline + - Wanda + - Craig + # - Gary + # - Rose + + - name: fine_model_but_with_a_no_good_very_long_name + columns: + - name: quite_long_column_name + tests: + # test truncation of really long test name with builtin + - unique diff --git a/test/integration/067_store_test_failures_tests/test_store_test_failures.py b/test/integration/067_store_test_failures_tests/test_store_test_failures.py new file mode 100644 index 00000000000..f203f659a29 --- /dev/null +++ b/test/integration/067_store_test_failures_tests/test_store_test_failures.py @@ -0,0 +1,110 @@ +from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile + +from dbt.task.test import TestTask +from dbt.task.list import ListTask + + +class TestStoreTestFailures(DBTIntegrationTest): + @property + def schema(self): + return "test_store_test_failures_067" + + @property + def models(self): + return "models" + + @property + def project_config(self): + return { + "config-version": 2, + "test-paths": ["tests"], + "seeds": { + "quote_columns": False, + "test": { + "expected": self.column_type_overrides() + }, + }, + } + + def column_type_overrides(self): + return {} + + def run_tests_store_failures_and_assert(self): + test_audit_schema = self.unique_schema() + "_dbt_test__audit" + + self.run_dbt(["seed"]) + self.run_dbt(["run"]) + # make sure this works idempotently + self.run_dbt(["test", "--store-failures"], expect_pass=False) + results = self.run_dbt(["test", "--store-failures"], expect_pass=False) + + # compare test results + actual = [(r.status, r.message) for r in results] + expected = [('pass', 0), ('pass', 0), ('pass', 0), ('pass', 0), + ('fail', 2), ('fail', 2), ('fail', 2), ('fail', 10),] + self.assertEqual(sorted(actual), sorted(expected)) + + # compare test results stored in database + self.assertTablesEqual("failing_test", "expected_failing_test", test_audit_schema) + self.assertTablesEqual("not_null_problematic_model_id", "expected_not_null_problematic_model_id", test_audit_schema) + self.assertTablesEqual("unique_problematic_model_id", "expected_unique_problematic_model_id", test_audit_schema) + self.assertTablesEqual("accepted_values_problematic_mo_c533ab4ca65c1a9dbf14f79ded49b628", "expected_accepted_values", test_audit_schema) + +class PostgresTestStoreTestFailures(TestStoreTestFailures): + @property + def schema(self): + return "067" # otherwise too long + truncated + + def column_type_overrides(self): + return { + "expected_unique_problematic_model_id": { + "+column_types": { + "n_records": "bigint", + }, + }, + "expected_accepted_values": { + "+column_types": { + "n_records": "bigint", + }, + }, + } + + @use_profile('postgres') + def test__postgres__store_and_assert(self): + self.run_tests_store_failures_and_assert() + +class RedshiftTestStoreTestFailures(TestStoreTestFailures): + def column_type_overrides(self): + return { + "expected_not_null_problematic_model_id": { + "+column_types": { + "email": "varchar(26)", + "first_name": "varchar(10)", + }, + }, + "expected_unique_problematic_model_id": { + "+column_types": { + "n_records": "bigint", + }, + }, + "expected_accepted_values": { + "+column_types": { + "value_field": "varchar(10)", + "n_records": "bigint", + }, + }, + } + + @use_profile('redshift') + def test__redshift__store_and_assert(self): + self.run_tests_store_failures_and_assert() + +class SnowflakeTestStoreTestFailures(TestStoreTestFailures): + @use_profile('snowflake') + def test__snowflake__store_and_assert(self): + self.run_tests_store_failures_and_assert() + +class BigQueryTestStoreTestFailures(TestStoreTestFailures): + @use_profile('bigquery') + def test__bigquery__store_and_assert(self): + self.run_tests_store_failures_and_assert() diff --git a/test/integration/067_store_test_failures_tests/tests/failing_test.sql b/test/integration/067_store_test_failures_tests/tests/failing_test.sql new file mode 100644 index 00000000000..1bb5ae5ba6e --- /dev/null +++ b/test/integration/067_store_test_failures_tests/tests/failing_test.sql @@ -0,0 +1 @@ +select * from {{ ref('fine_model') }} diff --git a/test/integration/067_store_test_failures_tests/tests/passing_test.sql b/test/integration/067_store_test_failures_tests/tests/passing_test.sql new file mode 100644 index 00000000000..15c9a7a642d --- /dev/null +++ b/test/integration/067_store_test_failures_tests/tests/passing_test.sql @@ -0,0 +1,2 @@ +select * from {{ ref('fine_model') }} +where false diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index 5d6e6a4d8d9..423fbdd6d24 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -318,7 +318,7 @@ def minimal_schema_test_dict(): 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'database': 'test_db', - 'schema': 'test_schema', + 'schema': 'dbt_test__audit', 'alias': 'bar', 'test_metadata': { 'name': 'foo', @@ -347,7 +347,7 @@ def basic_uncompiled_schema_test_node(): depends_on=DependsOn(), description='', database='test_db', - schema='test_schema', + schema='dbt_test__audit', alias='bar', tags=[], config=TestConfig(), @@ -379,7 +379,7 @@ def basic_compiled_schema_test_node(): deferred=False, description='', database='test_db', - schema='test_schema', + schema='dbt_test__audit', alias='bar', tags=[], config=TestConfig(severity='warn'), @@ -414,7 +414,7 @@ def basic_uncompiled_schema_test_dict(): 'depends_on': {'macros': [], 'nodes': []}, 'database': 'test_db', 'description': '', - 'schema': 'test_schema', + 'schema': 'dbt_test__audit', 'alias': 'bar', 'tags': [], 'config': { @@ -428,6 +428,7 @@ def basic_uncompiled_schema_test_dict(): 'tags': [], 'vars': {}, 'severity': 'ERROR', + 'schema': 'dbt_test__audit', }, 'deferred': False, 'docs': {'show': True}, @@ -463,7 +464,7 @@ def basic_compiled_schema_test_dict(): 'deferred': False, 'database': 'test_db', 'description': '', - 'schema': 'test_schema', + 'schema': 'dbt_test__audit', 'alias': 'bar', 'tags': [], 'config': { @@ -477,6 +478,7 @@ def basic_compiled_schema_test_dict(): 'tags': [], 'vars': {}, 'severity': 'warn', + 'schema': 'dbt_test__audit', }, 'docs': {'show': True}, diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 92d1568af06..978044dd9d0 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -987,6 +987,7 @@ def basic_parsed_schema_test_dict(): 'tags': [], 'vars': {}, 'severity': 'ERROR', + 'schema': 'dbt_test__audit', }, 'docs': {'show': True}, 'columns': {}, @@ -1059,7 +1060,8 @@ def complex_parsed_schema_test_dict(): 'tags': [], 'vars': {}, 'severity': 'WARN', - 'extra_key': 'extra value' + 'extra_key': 'extra value', + 'schema': 'dbt_test__audit', }, 'docs': {'show': False}, 'columns': { diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 582888ae469..c068754e46e 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -750,7 +750,7 @@ def test_basic(self): alias='test_1', name='test_1', database='test', - schema='analytics', + schema='dbt_test__audit', resource_type=NodeType.Test, unique_id='test.snowplow.test_1', fqn=['snowplow', 'data_test', 'test_1'],