Merge pull request #3318 from fishtown-analytics/fix/ephemeral-compil…

…ation

[moving fix to current release] Fix ephemeral model compilation

CHANGELOG.md

@@ -1,7 +1,9 @@
 ## dbt 0.20.0 (Release TBD)
 
-## dbt 0.20.0b1 (May 03, 2021)
+### Fixes
+- Fix compiled sql for ephemeral models ([#3317](https:/fishtown-analytics/dbt/issues/3317), [#3318](https:/fishtown-analytics/dbt/pull/3318))
 
+## dbt 0.20.0b1 (May 03, 2021)
 
 ### Fixes
 - Fix exit code from dbt debug not returning a failure when one of the tests fail ([#3017](https:/fishtown-analytics/dbt/issues/3017))

core/dbt/compilation.py

@@ -250,19 +250,19 @@ def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
 
  return str(parsed)
 
- # This method is called by the 'compile_node' method. Starting
- # from the node that it is passed in, it will recursively call
- # itself using the 'extra_ctes'. The 'ephemeral' models do
- # not produce SQL that is executed directly, instead they
- # are rolled up into the models that refer to them by
- # inserting CTEs into the SQL.
  def _recursively_prepend_ctes(
  self,
  model: NonSourceCompiledNode,
  manifest: Manifest,
  extra_context: Optional[Dict[str, Any]],
  ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
-
+ """This method is called by the 'compile_node' method. Starting
+ from the node that it is passed in, it will recursively call
+ itself using the 'extra_ctes'. The 'ephemeral' models do
+ not produce SQL that is executed directly, instead they
+ are rolled up into the models that refer to them by
+ inserting CTEs into the SQL.
+ """
  if model.compiled_sql is None:
  raise RuntimeException(
  'Cannot inject ctes into an unparsed node', model
@@ -320,19 +320,19 @@ def _recursively_prepend_ctes(
  _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
  new_cte_name = self.add_ephemeral_prefix(cte_model.name)
- sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+ rendered_sql = (
+ cte_model._pre_injected_sql or cte_model.compiled_sql
+ )
+ sql = f' {new_cte_name} as (\n{rendered_sql}\n)'
 
- _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
+  _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
- # We don't save injected_sql into compiled sql for ephemeral models
- # because it will cause problems with processing of subsequent models.
- # Ephemeral models do not produce executable SQL of their own.
- if not model.is_ephemeral_model:
- injected_sql = self._inject_ctes_into_sql(
- model.compiled_sql,
- prepended_ctes,
- )
- model.compiled_sql = injected_sql
+ injected_sql = self._inject_ctes_into_sql(
+ model.compiled_sql,
+ prepended_ctes,
+ )
+ model._pre_injected_sql = model.compiled_sql
+ model.compiled_sql = injected_sql
  model.extra_ctes_injected = True
  model.extra_ctes = prepended_ctes
  model.validate(model.to_dict(omit_none=True))
@@ -449,18 +449,19 @@ def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
  )
  return node
 
- # This is the main entry point into this code. It's called by
- # CompileRunner.compile, GenericRPCRunner.compile, and
- # RunTask.get_hook_sql. It calls '_compile_node' to convert
- # the node into a compiled node, and then calls the
- # recursive method to "prepend" the ctes.
  def compile_node(
  self,
  node: ManifestNode,
  manifest: Manifest,
  extra_context: Optional[Dict[str, Any]] = None,
  write: bool = True,
  ) -> NonSourceCompiledNode:
+ """This is the main entry point into this code. It's called by
+ CompileRunner.compile, GenericRPCRunner.compile, and
+ RunTask.get_hook_sql. It calls '_compile_node' to convert
+ the node into a compiled node, and then calls the
+ recursive method to "prepend" the ctes.
+ """
  node = self._compile_node(node, manifest, extra_context)
 
  node, _ = self._recursively_prepend_ctes(

core/dbt/contracts/graph/compiled.py

@@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
  extra_ctes_injected: bool = False
  extra_ctes: List[InjectedCTE] = field(default_factory=list)
  relation_name: Optional[str] = None
+ _pre_injected_sql: Optional[str] = None
 
  def set_cte(self, cte_id: str, sql: str):
  """This is the equivalent of what self.extra_ctes[cte_id] = sql would
@@ -55,6 +56,12 @@ def set_cte(self, cte_id: str, sql: str):
  else:
  self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
 
+ def __post_serialize__(self, dct):
+ dct = super().__post_serialize__(dct)
+ if '_pre_injected_sql' in dct:
+ del dct['_pre_injected_sql']
+ return dct
+
 
 @dataclass
 class CompiledAnalysisNode(CompiledNode):