Merge pull request #2623 from fishtown-analytics/asset-paths

Compile assets as part of docs generate

CHANGELOG.md

@@ -11,6 +11,7 @@
 ### Fixes
 - Adapter plugins can once again override plugins defined in core ([#2548](https:/fishtown-analytics/dbt/issues/2548), [#2590](https:/fishtown-analytics/dbt/pull/2590))
 - Added `--selector` argument and support for `selectors.yml` file to define selection mechanisms. ([#2172](https:/fishtown-analytics/dbt/issues/2172), [#2640](https:/fishtown-analytics/dbt/pull/2640))
+- Compile assets as part of docs generate ([#2072](https:/fishtown-analytics/dbt/issues/2072), [#2623](https:/fishtown-analytics/dbt/pull/2623))
 
 Contributors:
 - [@brunomurino](https:/brunomurino) ([#2437](https:/fishtown-analytics/dbt/pull/2581))

core/dbt/config/project.py

@@ -304,6 +304,7 @@ class Project:
  test_paths: List[str]
  analysis_paths: List[str]
  docs_paths: List[str]
+ asset_paths: List[str]
  target_path: str
  snapshot_paths: List[str]
  clean_targets: List[str]
@@ -412,6 +413,7 @@ def from_project_config(
  )
 
  docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
+ asset_paths: List[str] = value_or(cfg.asset_paths, [])
  target_path: str = value_or(cfg.target_path, 'target')
  clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
  log_path: str = value_or(cfg.log_path, 'logs')
@@ -491,6 +493,7 @@ def from_project_config(
  test_paths=test_paths,
  analysis_paths=analysis_paths,
  docs_paths=docs_paths,
+ asset_paths=asset_paths,
  target_path=target_path,
  snapshot_paths=snapshot_paths,
  clean_targets=clean_targets,
@@ -544,6 +547,7 @@ def to_project_config(self, with_packages=False):
  'test-paths': self.test_paths,
  'analysis-paths': self.analysis_paths,
  'docs-paths': self.docs_paths,
+ 'asset-paths': self.asset_paths,
  'target-path': self.target_path,
  'snapshot-paths': self.snapshot_paths,
  'clean-targets': self.clean_targets,

core/dbt/config/runtime.py

@@ -93,6 +93,7 @@ def from_parts(
  test_paths=project.test_paths,
  analysis_paths=project.analysis_paths,
  docs_paths=project.docs_paths,
+ asset_paths=project.asset_paths,
  target_path=project.target_path,
  snapshot_paths=project.snapshot_paths,
  clean_targets=project.clean_targets,
@@ -490,6 +491,7 @@ def from_parts(
  test_paths=project.test_paths,
  analysis_paths=project.analysis_paths,
  docs_paths=project.docs_paths,
+ asset_paths=project.asset_paths,
  target_path=project.target_path,
  snapshot_paths=project.snapshot_paths,
  clean_targets=project.clean_targets,

core/dbt/contracts/project.py

@@ -164,6 +164,7 @@ class ProjectV1(HyphenatedJsonSchemaMixin, Replaceable):
  test_paths: Optional[List[str]] = None
  analysis_paths: Optional[List[str]] = None
  docs_paths: Optional[List[str]] = None
+ asset_paths: Optional[List[str]] = None
  target_path: Optional[str] = None
  snapshot_paths: Optional[List[str]] = None
  clean_targets: Optional[List[str]] = None
@@ -204,6 +205,7 @@ class ProjectV2(HyphenatedJsonSchemaMixin, Replaceable):
  test_paths: Optional[List[str]] = None
  analysis_paths: Optional[List[str]] = None
  docs_paths: Optional[List[str]] = None
+ asset_paths: Optional[List[str]] = None
  target_path: Optional[str] = None
  snapshot_paths: Optional[List[str]] = None
  clean_targets: Optional[List[str]] = None

core/dbt/task/generate.py

@@ -230,6 +230,17 @@ def run(self) -> CatalogResults:
  DOCS_INDEX_FILE_PATH,
  os.path.join(self.config.target_path, 'index.html'))
 
+ for asset_path in self.config.asset_paths:
+ to_asset_path = os.path.join(self.config.target_path, asset_path)
+
+ if os.path.exists(to_asset_path):
+ shutil.rmtree(to_asset_path)
+
+ if os.path.exists(asset_path):
+ shutil.copytree(
+ asset_path,
+ to_asset_path)
+
  if self.manifest is None:
  raise InternalException(
  'self.manifest was None in run!'

test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -3072,6 +3072,17 @@ def test__postgres_references(self):
  self.verify_manifest(self.expected_postgres_references_manifest())
  self.verify_run_results(self.expected_postgres_references_run_results())
 
+ @use_profile('postgres')
+ def test_postgres_asset_paths_copied(self):
+ self.run_and_generate(
+ {'asset-paths': [self.dir('assets'), self.dir('non-existent-assets')]},
+ )
+
+ assert os.path.exists('./target/assets')
+ assert os.path.exists('./target/assets/lorem-ipsum.txt')
+
+ assert not os.path.exists('./target/non-existent-assets')
+
  @use_profile('snowflake')
  def test__snowflake__run_and_generate(self):
  self.run_and_generate()

test/unit/test_config.py

@@ -580,6 +580,7 @@ def test_defaults(self):
  self.assertEqual(project.test_paths, ['test'])
  self.assertEqual(project.analysis_paths, [])
  self.assertEqual(project.docs_paths, ['models', 'data', 'snapshots', 'macros'])
+ self.assertEqual(project.asset_paths, [])
  self.assertEqual(project.target_path, 'target')
  self.assertEqual(project.clean_targets, ['target'])
  self.assertEqual(project.log_path, 'logs')
@@ -636,6 +637,7 @@ def test_all_overrides(self):
  'test-paths': ['other-test'],
  'analysis-paths': ['analysis'],
  'docs-paths': ['docs'],
+ 'asset-paths': ['other-assets'],
  'target-path': 'other-target',
  'clean-targets': ['another-target'],
  'log-path': 'other-logs',
@@ -700,6 +702,7 @@ def test_all_overrides(self):
  self.assertEqual(project.test_paths, ['other-test'])
  self.assertEqual(project.analysis_paths, ['analysis'])
  self.assertEqual(project.docs_paths, ['docs'])
+ self.assertEqual(project.asset_paths, ['other-assets'])
  self.assertEqual(project.target_path, 'other-target')
  self.assertEqual(project.clean_targets, ['another-target'])
  self.assertEqual(project.log_path, 'other-logs')
@@ -1186,6 +1189,7 @@ def test_from_args(self):
  self.assertEqual(config.test_paths, ['test'])
  self.assertEqual(config.analysis_paths, [])
  self.assertEqual(config.docs_paths, ['models', 'data', 'snapshots', 'macros'])
+ self.assertEqual(config.asset_paths, [])
  self.assertEqual(config.target_path, 'target')
  self.assertEqual(config.clean_targets, ['target'])
  self.assertEqual(config.log_path, 'logs')