From f7fd854d1197fa14c277d844b00dbc8e1e48937e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 25 Feb 2020 12:15:33 -0700 Subject: [PATCH] Allow search in macro and analysis paths add tests --- CHANGELOG.md | 1 + core/dbt/config/project.py | 15 +++-- .../029_docs_generate_tests/macros/macro.md | 8 +++ .../029_docs_generate_tests/macros/schema.yml | 10 +++ .../ref_models/docs.md | 8 --- .../ref_models/schema.yml | 10 --- .../test_docs_generate.py | 64 +++++++++++++++++-- 7 files changed, 87 insertions(+), 29 deletions(-) create mode 100644 test/integration/029_docs_generate_tests/macros/macro.md create mode 100644 test/integration/029_docs_generate_tests/macros/schema.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ba27d120b..eded399db82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Add a "docs" field to models, with a "show" subfield ([#1671](https://github.com/fishtown-analytics/dbt/issues/1671), [#2107](https://github.com/fishtown-analytics/dbt/pull/2107)) - Add a dbt-{dbt_version} user agent field to the bigquery connector ([#2121](https://github.com/fishtown-analytics/dbt/issues/2121), [#2146](https://github.com/fishtown-analytics/dbt/pull/2146)) - Add support for `generate_database_name` macro ([#1695](https://github.com/fishtown-analytics/dbt/issues/1695), [#2143](https://github.com/fishtown-analytics/dbt/pull/2143)) +- Expand the search path for schema.yml (an dby extension, the default docs path) to include macro-paths and analysis-paths (in addition to source-paths, data-paths, and snapshot-paths) ([#2155](https://github.com/fishtown-analytics/dbt/issues/2155), [#2159](https://github.com/fishtown-analytics/dbt/pull/2159)) ### Fixes - Fix issue where dbt did not give an error in the presence of duplicate doc names ([#2054](https://github.com/fishtown-analytics/dbt/issues/2054), [#2080](https://github.com/fishtown-analytics/dbt/pull/2080)) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index ab2e14d37fd..77b49678f53 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -160,9 +160,14 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]: def _all_source_paths( - source_paths: List[str], data_paths: List[str], snapshot_paths: List[str] + source_paths: List[str], + data_paths: List[str], + snapshot_paths: List[str], + analysis_paths: List[str], + macro_paths: List[str], ) -> List[str]: - return list(chain(source_paths, data_paths, snapshot_paths)) + return list(chain(source_paths, data_paths, snapshot_paths, analysis_paths, + macro_paths)) T = TypeVar('T') @@ -244,7 +249,8 @@ class Project: @property def all_source_paths(self) -> List[str]: return _all_source_paths( - self.source_paths, self.data_paths, self.snapshot_paths + self.source_paths, self.data_paths, self.snapshot_paths, + self.analysis_paths, self.macro_paths ) @staticmethod @@ -323,7 +329,8 @@ def from_project_config( snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ['snapshots']) all_source_paths: List[str] = _all_source_paths( - source_paths, data_paths, snapshot_paths + source_paths, data_paths, snapshot_paths, analysis_paths, + macro_paths ) docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths) diff --git a/test/integration/029_docs_generate_tests/macros/macro.md b/test/integration/029_docs_generate_tests/macros/macro.md new file mode 100644 index 00000000000..6a48ffc1b37 --- /dev/null +++ b/test/integration/029_docs_generate_tests/macros/macro.md @@ -0,0 +1,8 @@ + +{% docs macro_info %} +My custom test that I wrote that does nothing +{% enddocs %} + +{% docs macro_arg_info %} +The model for my custom test +{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/macros/schema.yml b/test/integration/029_docs_generate_tests/macros/schema.yml new file mode 100644 index 00000000000..6b33f47307e --- /dev/null +++ b/test/integration/029_docs_generate_tests/macros/schema.yml @@ -0,0 +1,10 @@ +version: 2 +macros: + - name: test_nothing + description: "{{ doc('macro_info') }}" + meta: + some_key: 100 + arguments: + - name: model + type: Relation + description: "{{ doc('macro_arg_info') }}" diff --git a/test/integration/029_docs_generate_tests/ref_models/docs.md b/test/integration/029_docs_generate_tests/ref_models/docs.md index 595444ec933..1918e825b44 100644 --- a/test/integration/029_docs_generate_tests/ref_models/docs.md +++ b/test/integration/029_docs_generate_tests/ref_models/docs.md @@ -25,11 +25,3 @@ My table {% docs column_info %} An ID field {% enddocs %} - -{% docs macro_info %} -My custom test that I wrote that does nothing -{% enddocs %} - -{% docs macro_arg_info %} -The model for my custom test -{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/ref_models/schema.yml b/test/integration/029_docs_generate_tests/ref_models/schema.yml index 997eccb60c5..3cc4c7aac10 100644 --- a/test/integration/029_docs_generate_tests/ref_models/schema.yml +++ b/test/integration/029_docs_generate_tests/ref_models/schema.yml @@ -29,13 +29,3 @@ sources: columns: - name: id description: "{{ doc('column_info') }}" - -macros: - - name: test_nothing - description: "{{ doc('macro_info') }}" - meta: - some_key: 100 - arguments: - - name: model - type: Relation - description: "{{ doc('macro_arg_info') }}" diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 649d27c0371..40134b1152b 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1219,7 +1219,9 @@ def expected_seeded_manifest(self, model_database=None): 'test.test.unique_model_id': [], }, 'docs': { - 'dbt.__overview__': ANY + 'dbt.__overview__': ANY, + 'test.macro_info': ANY, + 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', @@ -1239,6 +1241,29 @@ def expected_seeded_manifest(self, model_database=None): 'patches': [], 'macro_patches': [], }, + normalize('macros/macro.md'): { + 'checksum': self._checksum_file('macros/macro.md'), + 'docs': [ + 'test.macro_info', + 'test.macro_arg_info', + ], + 'macros': [], + 'nodes': [], + 'patches': [], + 'path': self._path_to('macros', 'macro.md'), + 'sources': [], + 'macro_patches': [], + }, + normalize('macros/schema.yml'): { + 'path': self._path_to('macros', 'schema.yml'), + 'checksum': self._checksum_file('macros/schema.yml'), + 'docs': [], + 'macros': [], + 'nodes': [], + 'patches': [], + 'sources': [], + 'macro_patches': [['test', 'test_nothing']], + }, normalize('models/model.sql'): { 'path': self._path_to('models', 'model.sql'), 'checksum': self._checksum_file('models/model.sql'), @@ -1704,9 +1729,9 @@ def expected_postgres_references_manifest(self, model_database=None): 'block_contents': 'My custom test that I wrote that does nothing', 'file_contents': macro_info, 'name': 'macro_info', - 'original_file_path': docs_path, + 'original_file_path': self.dir('macros/macro.md'), 'package_name': 'test', - 'path': 'docs.md', + 'path': 'macro.md', 'root_path': self.test_root_dir, 'unique_id': 'test.macro_info', }, @@ -1714,9 +1739,9 @@ def expected_postgres_references_manifest(self, model_database=None): 'block_contents': 'The model for my custom test', 'file_contents': macro_arg_info, 'name': 'macro_arg_info', - 'original_file_path': docs_path, + 'original_file_path': self.dir('macros/macro.md'), 'package_name': 'test', - 'path': 'docs.md', + 'path': 'macro.md', 'root_path': self.test_root_dir, 'unique_id': 'test.macro_arg_info', }, @@ -1806,13 +1831,24 @@ def expected_postgres_references_manifest(self, model_database=None): 'test.source_info', 'test.table_info', 'test.column_info', + ], + 'macros': [], + 'nodes': [], + 'patches': [], + 'path': self._path_to('ref_models', 'docs.md'), + 'sources': [], + 'macro_patches': [], + }, + normalize('macros/macro.md'): { + 'checksum': self._checksum_file('macros/macro.md'), + 'docs': [ 'test.macro_info', 'test.macro_arg_info', ], 'macros': [], 'nodes': [], 'patches': [], - 'path': self._path_to('ref_models', 'docs.md'), + 'path': self._path_to('macros', 'macro.md'), 'sources': [], 'macro_patches': [], }, @@ -1824,6 +1860,16 @@ def expected_postgres_references_manifest(self, model_database=None): 'patches': ['ephemeral_summary', 'view_summary'], 'path': self._path_to('ref_models', 'schema.yml'), 'sources': ['source.test.my_source.my_table'], + 'macro_patches': [], + }, + normalize('macros/schema.yml'): { + 'path': self._path_to('macros', 'schema.yml'), + 'checksum': self._checksum_file('macros/schema.yml'), + 'docs': [], + 'macros': [], + 'nodes': [], + 'patches': [], + 'sources': [], 'macro_patches': [['test', 'test_nothing']], }, normalize('seed/schema.yml'): { @@ -1850,7 +1896,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'meta': { 'some_key': 100, }, - 'patch_path': self.dir('ref_models/schema.yml'), + 'patch_path': self.dir('macros/schema.yml'), 'resource_type': 'macro', 'unique_id': 'macro.test.test_nothing', 'tags': [], @@ -2256,6 +2302,8 @@ def expected_bigquery_complex_manifest(self): }, 'docs': { 'dbt.__overview__': ANY, + 'test.macro_info': ANY, + 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', @@ -2557,6 +2605,8 @@ def expected_redshift_incremental_view_manifest(self): }, 'docs': { 'dbt.__overview__': ANY, + 'test.macro_info': ANY, + 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6',