Merge pull request #2349 from franloza/fix/2173

Use original path to mimick resource subdirectory structure
dbt-labs · May 4, 2020 · bd9a132 · bd9a132
2 parents 58c95bb + 9eb4506
commit bd9a132
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -39,6 +39,7 @@
 - On `run_cli` API calls that are passed `--vars` differing from the server's `--vars`, the RPC server rebuilds the manifest for that call. ([#2265](https:/fishtown-analytics/dbt/issues/2265), [#2363](https:/fishtown-analytics/dbt/pull/2363))
 - Fix "Object of type Decimal is not JSON serializable" error when BigQuery queries returned numeric types in nested data structures ([#2336](https:/fishtown-analytics/dbt/issues/2336), [#2348](https:/fishtown-analytics/dbt/pull/2348))
 - No longer query the information_schema.schemata view on bigquery ([#2320](https:/fishtown-analytics/dbt/issues/2320), [#2382](https:/fishtown-analytics/dbt/pull/2382))
+- Preserve original subdirectory structure in compiled files. ([#2173](https:/fishtown-analytics/dbt/issues/2173), [#2349](https:/fishtown-analytics/dbt/pull/2349))
 - Add support for `sql_header` config in incremental models ([#2136](https:/fishtown-analytics/dbt/issues/2136), [#2200](https:/fishtown-analytics/dbt/pull/2200))
 - The ambiguous alias check now examines the node's database value as well as the schema/identifier ([#2326](https:/fishtown-analytics/dbt/issues/2326), [#2387](https:/fishtown-analytics/dbt/pull/2387))
 - Postgres array types can now be returned via `run_query` macro calls ([#2337](https:/fishtown-analytics/dbt/issues/2337), [#2376](https:/fishtown-analytics/dbt/pull/2376))
@@ -58,6 +59,7 @@ Contributors:
  - [@sumanau7](https:/sumanau7) ([#2279](https:/fishtown-analytics/dbt/pull/2279), [#2263](https:/fishtown-analytics/dbt/pull/2263), [#2297](https:/fishtown-analytics/dbt/pull/2297))
  - [@nickwu241](https:/nickwu241) [#2339](https:/fishtown-analytics/dbt/issues/2339)
  - [@Fokko](https:/Fokko) [#2361](https:/fishtown-analytics/dbt/pull/2361)
+ - [@franloza](https:/franloza) [#2349](https:/fishtown-analytics/dbt/pull/2349)
  - [@sethwoodworth](https:/sethwoodworth) [#2389](https:/fishtown-analytics/dbt/pull/2389)
 
 ## dbt 0.16.1 (April 14, 2020)

diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py
@@ -158,8 +158,15 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
  build_path: Optional[str] = None
 
  def write_node(self, target_path: str, subdirectory: str, payload: str):
+ if (os.path.basename(self.path) ==
+ os.path.basename(self.original_file_path)):
+ # One-to-one relationship of nodes to files.
+ path = self.original_file_path
+ else:
+ # Many-to-one relationship of nodes to files.
+ path = os.path.join(self.original_file_path, self.path)
  full_path = os.path.join(
- target_path, subdirectory, self.package_name, self.path
+ target_path, subdirectory, self.package_name, path
  )
 
  write_file(full_path, payload)

diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -903,7 +903,7 @@ def expected_seeded_manifest(self, model_database=None):
  return {
  'nodes': {
  'model.test.model': {
- 'build_path': Normalized('target/compiled/test/model.sql'),
+ 'build_path': Normalized('target/compiled/test/models/model.sql'),
  'name': 'model',
  'root_path': self.test_root_dir,
  'resource_type': 'model',
@@ -1050,7 +1050,7 @@ def expected_seeded_manifest(self, model_database=None):
  },
  'test.test.not_null_model_id': {
  'alias': 'not_null_model_id',
- 'build_path': Normalized('target/compiled/test/schema_test/not_null_model_id.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/not_null_model_id.sql'),
  'column_name': 'id',
  'columns': {},
  'config': {
@@ -1103,7 +1103,7 @@ def expected_seeded_manifest(self, model_database=None):
  },
  'test.test.test_nothing_model_': {
  'alias': 'test_nothing_model_',
- 'build_path': Normalized('target/compiled/test/schema_test/test_nothing_model_.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/test_nothing_model_.sql'),
  'column_name': None,
  'columns': {},
  'config': {
@@ -1155,7 +1155,7 @@ def expected_seeded_manifest(self, model_database=None):
  },
  'test.test.unique_model_id': {
  'alias': 'unique_model_id',
- 'build_path': Normalized('target/compiled/test/schema_test/unique_model_id.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/unique_model_id.sql'),
  'column_name': 'id',
  'columns': {},
  'config': {
@@ -1274,7 +1274,7 @@ def expected_postgres_references_manifest(self, model_database=None):
  'nodes': {
  'model.test.ephemeral_copy': {
  'alias': 'ephemeral_copy',
- 'build_path': Normalized('target/compiled/test/ephemeral_copy.sql'),
+ 'build_path': Normalized('target/compiled/test/ref_models/ephemeral_copy.sql'),
  'columns': {},
  'config': {
  'column_types': {},
@@ -1320,7 +1320,7 @@ def expected_postgres_references_manifest(self, model_database=None):
  },
  'model.test.ephemeral_summary': {
  'alias': 'ephemeral_summary',
- 'build_path': Normalized('target/compiled/test/ephemeral_summary.sql'),
+ 'build_path': Normalized('target/compiled/test/ref_models/ephemeral_summary.sql'),
  'columns': {
  'first_name': {
  'description': 'The first name being summarized',
@@ -1383,7 +1383,7 @@ def expected_postgres_references_manifest(self, model_database=None):
  },
  'model.test.view_summary': {
  'alias': 'view_summary',
- 'build_path': Normalized('target/compiled/test/view_summary.sql'),
+ 'build_path': Normalized('target/compiled/test/ref_models/view_summary.sql'),
  'columns': {
  'first_name': {
  'description': 'The first name being summarized',
@@ -1729,7 +1729,7 @@ def expected_bigquery_complex_manifest(self):
  'sources': [],
  'depends_on': {'macros': [], 'nodes': ['seed.test.seed']},
  'fqn': ['test', 'clustered'],
- 'build_path': Normalized('target/compiled/test/clustered.sql'),
+ 'build_path': Normalized('target/compiled/test/bq_models/clustered.sql'),
  'name': 'clustered',
  'original_file_path': clustered_sql_path,
  'package_name': 'test',
@@ -1791,7 +1791,7 @@ def expected_bigquery_complex_manifest(self):
  },
  'model.test.multi_clustered': {
  'alias': 'multi_clustered',
- 'build_path': Normalized('target/compiled/test/multi_clustered.sql'),
+ 'build_path': Normalized('target/compiled/test/bq_models/multi_clustered.sql'),
  'config': {
  'cluster_by': ['first_name', 'email'],
  'column_types': {},
@@ -1869,7 +1869,7 @@ def expected_bigquery_complex_manifest(self):
  },
  'model.test.nested_view': {
  'alias': 'nested_view',
- 'build_path': Normalized('target/compiled/test/nested_view.sql'),
+ 'build_path': Normalized('target/compiled/test/bq_models/nested_view.sql'),
  'config': {
  'column_types': {},
  'enabled': True,
@@ -1948,7 +1948,7 @@ def expected_bigquery_complex_manifest(self):
  },
  'model.test.nested_table': {
  'alias': 'nested_table',
- 'build_path': Normalized('target/compiled/test/nested_table.sql'),
+ 'build_path': Normalized('target/compiled/test/bq_models/nested_table.sql'),
  'config': {
  'column_types': {},
  'enabled': True,
@@ -2131,7 +2131,7 @@ def expected_redshift_incremental_view_manifest(self):
  return {
  'nodes': {
  'model.test.model': {
- 'build_path': Normalized('target/compiled/test/model.sql'),
+ 'build_path': Normalized('target/compiled/test/rs_models/model.sql'),
  'name': 'model',
  'root_path': self.test_root_dir,
  'resource_type': 'model',
@@ -2392,7 +2392,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
  'node': {
  'alias': 'model',
  'build_path': Normalized(
- 'target/compiled/test/model.sql'
+ 'target/compiled/test/models/model.sql'
  ),
  'columns': {
  'id': {
@@ -2559,7 +2559,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
  'warn': None,
  'node': {
  'alias': 'not_null_model_id',
- 'build_path': Normalized('target/compiled/test/schema_test/not_null_model_id.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/not_null_model_id.sql'),
  'column_name': 'id',
  'columns': {},
  'compiled': True,
@@ -2622,7 +2622,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
  'warn': None,
  'node': {
  'alias': 'test_nothing_model_',
- 'build_path': Normalized('target/compiled/test/schema_test/test_nothing_model_.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/test_nothing_model_.sql'),
  'column_name': None,
  'columns': {},
  'compiled': True,
@@ -2684,7 +2684,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False,
  'warn': None,
  'node': {
  'alias': 'unique_model_id',
- 'build_path': Normalized('target/compiled/test/schema_test/unique_model_id.sql'),
+ 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/unique_model_id.sql'),
  'column_name': 'id',
  'columns': {},
  'compiled': True,
@@ -2774,7 +2774,7 @@ def expected_postgres_references_run_results(self):
  'node': {
  'alias': 'ephemeral_summary',
  'build_path': Normalized(
- 'target/compiled/test/ephemeral_summary.sql'
+ 'target/compiled/test/ref_models/ephemeral_summary.sql'
  ),
  'columns': {
  'first_name': {
@@ -2853,7 +2853,7 @@ def expected_postgres_references_run_results(self):
  'node': {
  'alias': 'view_summary',
  'build_path': Normalized(
- 'target/compiled/test/view_summary.sql'
+ 'target/compiled/test/ref_models/view_summary.sql'
  ),
  'alias': 'view_summary',
  'columns': {