From a3a1b143cffee1ebe36527ca33a80d800e567308 Mon Sep 17 00:00:00 2001 From: Claire Carroll Date: Tue, 21 Jul 2020 10:51:30 -0400 Subject: [PATCH] Compile assets as part of docs generate --- CHANGELOG.md | 1 + core/dbt/config/project.py | 4 ++++ core/dbt/config/runtime.py | 2 ++ core/dbt/contracts/project.py | 2 ++ core/dbt/task/generate.py | 11 +++++++++++ .../029_docs_generate_tests/assets/lorem-ipsum.txt | 1 + .../029_docs_generate_tests/test_docs_generate.py | 11 +++++++++++ test/unit/test_config.py | 4 ++++ 8 files changed, 36 insertions(+) create mode 100644 test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index f39b60afd18..6420885fbdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Fixes - Adapter plugins can once again override plugins defined in core ([#2548](https://github.com/fishtown-analytics/dbt/issues/2548), [#2590](https://github.com/fishtown-analytics/dbt/pull/2590)) - Added `--selector` argument and support for `selectors.yml` file to define selection mechanisms. ([#2172](https://github.com/fishtown-analytics/dbt/issues/2172), [#2640](https://github.com/fishtown-analytics/dbt/pull/2640)) +- Compile assets as part of docs generate ([#2072](https://github.com/fishtown-analytics/dbt/issues/2072), [#2623](https://github.com/fishtown-analytics/dbt/pull/2623)) Contributors: - [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581)) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 83c787625d8..f16941533bc 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -304,6 +304,7 @@ class Project: test_paths: List[str] analysis_paths: List[str] docs_paths: List[str] + asset_paths: List[str] target_path: str snapshot_paths: List[str] clean_targets: List[str] @@ -412,6 +413,7 @@ def from_project_config( ) docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths) + asset_paths: List[str] = value_or(cfg.asset_paths, []) target_path: str = value_or(cfg.target_path, 'target') clean_targets: List[str] = value_or(cfg.clean_targets, [target_path]) log_path: str = value_or(cfg.log_path, 'logs') @@ -491,6 +493,7 @@ def from_project_config( test_paths=test_paths, analysis_paths=analysis_paths, docs_paths=docs_paths, + asset_paths=asset_paths, target_path=target_path, snapshot_paths=snapshot_paths, clean_targets=clean_targets, @@ -544,6 +547,7 @@ def to_project_config(self, with_packages=False): 'test-paths': self.test_paths, 'analysis-paths': self.analysis_paths, 'docs-paths': self.docs_paths, + 'asset-paths': self.asset_paths, 'target-path': self.target_path, 'snapshot-paths': self.snapshot_paths, 'clean-targets': self.clean_targets, diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 7ff2c8b9a23..e11471c9830 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -93,6 +93,7 @@ def from_parts( test_paths=project.test_paths, analysis_paths=project.analysis_paths, docs_paths=project.docs_paths, + asset_paths=project.asset_paths, target_path=project.target_path, snapshot_paths=project.snapshot_paths, clean_targets=project.clean_targets, @@ -490,6 +491,7 @@ def from_parts( test_paths=project.test_paths, analysis_paths=project.analysis_paths, docs_paths=project.docs_paths, + asset_paths=project.asset_paths, target_path=project.target_path, snapshot_paths=project.snapshot_paths, clean_targets=project.clean_targets, diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index b5a2a14f61e..e59828add8e 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -164,6 +164,7 @@ class ProjectV1(HyphenatedJsonSchemaMixin, Replaceable): test_paths: Optional[List[str]] = None analysis_paths: Optional[List[str]] = None docs_paths: Optional[List[str]] = None + asset_paths: Optional[List[str]] = None target_path: Optional[str] = None snapshot_paths: Optional[List[str]] = None clean_targets: Optional[List[str]] = None @@ -204,6 +205,7 @@ class ProjectV2(HyphenatedJsonSchemaMixin, Replaceable): test_paths: Optional[List[str]] = None analysis_paths: Optional[List[str]] = None docs_paths: Optional[List[str]] = None + asset_paths: Optional[List[str]] = None target_path: Optional[str] = None snapshot_paths: Optional[List[str]] = None clean_targets: Optional[List[str]] = None diff --git a/core/dbt/task/generate.py b/core/dbt/task/generate.py index f973323e264..57fc8dde8ab 100644 --- a/core/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -230,6 +230,17 @@ def run(self) -> CatalogResults: DOCS_INDEX_FILE_PATH, os.path.join(self.config.target_path, 'index.html')) + for asset_path in self.config.asset_paths: + to_asset_path = os.path.join(self.config.target_path, asset_path) + + if os.path.exists(to_asset_path): + shutil.rmtree(to_asset_path) + + if os.path.exists(asset_path): + shutil.copytree( + asset_path, + to_asset_path) + if self.manifest is None: raise InternalException( 'self.manifest was None in run!' diff --git a/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt b/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt new file mode 100644 index 00000000000..cee7a927c5f --- /dev/null +++ b/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 7c959eb0fb6..8ea973e31bf 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -3072,6 +3072,17 @@ def test__postgres_references(self): self.verify_manifest(self.expected_postgres_references_manifest()) self.verify_run_results(self.expected_postgres_references_run_results()) + @use_profile('postgres') + def test_postgres_asset_paths_copied(self): + self.run_and_generate( + {'asset-paths': [self.dir('assets'), self.dir('non-existent-assets')]}, + ) + + assert os.path.exists('./target/assets') + assert os.path.exists('./target/assets/lorem-ipsum.txt') + + assert not os.path.exists('./target/non-existent-assets') + @use_profile('snowflake') def test__snowflake__run_and_generate(self): self.run_and_generate() diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 058f2cf233d..fc73e1981ff 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -580,6 +580,7 @@ def test_defaults(self): self.assertEqual(project.test_paths, ['test']) self.assertEqual(project.analysis_paths, []) self.assertEqual(project.docs_paths, ['models', 'data', 'snapshots', 'macros']) + self.assertEqual(project.asset_paths, []) self.assertEqual(project.target_path, 'target') self.assertEqual(project.clean_targets, ['target']) self.assertEqual(project.log_path, 'logs') @@ -636,6 +637,7 @@ def test_all_overrides(self): 'test-paths': ['other-test'], 'analysis-paths': ['analysis'], 'docs-paths': ['docs'], + 'asset-paths': ['other-assets'], 'target-path': 'other-target', 'clean-targets': ['another-target'], 'log-path': 'other-logs', @@ -700,6 +702,7 @@ def test_all_overrides(self): self.assertEqual(project.test_paths, ['other-test']) self.assertEqual(project.analysis_paths, ['analysis']) self.assertEqual(project.docs_paths, ['docs']) + self.assertEqual(project.asset_paths, ['other-assets']) self.assertEqual(project.target_path, 'other-target') self.assertEqual(project.clean_targets, ['another-target']) self.assertEqual(project.log_path, 'other-logs') @@ -1186,6 +1189,7 @@ def test_from_args(self): self.assertEqual(config.test_paths, ['test']) self.assertEqual(config.analysis_paths, []) self.assertEqual(config.docs_paths, ['models', 'data', 'snapshots', 'macros']) + self.assertEqual(config.asset_paths, []) self.assertEqual(config.target_path, 'target') self.assertEqual(config.clean_targets, ['target']) self.assertEqual(config.log_path, 'logs')