add generate_database_name macro (#1695)

CHANGELOG.md

@@ -2,10 +2,12 @@
 
 ### Breaking changes
 - Arguments to source tests are not parsed in the config-rendering context, and are passed as their literal unparsed values to macros ([#2150](https:/fishtown-analytics/dbt/pull/2150))
+- `generate_schema_name` macros that accept a single argument are no longer supported ([#2143](https:/fishtown-analytics/dbt/pull/2143))
 
 ### Features
 - Add a "docs" field to models, with a "show" subfield ([#1671](https:/fishtown-analytics/dbt/issues/1671), [#2107](https:/fishtown-analytics/dbt/pull/2107))
 - Add a dbt-{dbt_version} user agent field to the bigquery connector ([#2121](https:/fishtown-analytics/dbt/issues/2121), [#2146](https:/fishtown-analytics/dbt/pull/2146))
+- Add support for `generate_database_name` macro ([#1695](https:/fishtown-analytics/dbt/issues/1695), [#2143](https:/fishtown-analytics/dbt/pull/2143))
 
 ### Fixes
 - Fix issue where dbt did not give an error in the presence of duplicate doc names ([#2054](https:/fishtown-analytics/dbt/issues/2054), [#2080](https:/fishtown-analytics/dbt/pull/2080))

core/dbt/contracts/graph/manifest.py

@@ -541,21 +541,21 @@ def filter(candidate: MacroCandidate) -> bool:
  return candidates.last()
 
  def find_generate_macro_by_name(
- self, name: str, root_project_name: str
+ self, component: str, root_project_name: str
  ) -> Optional[ParsedMacro]:
  """
  The `generate_X_name` macros are similar to regular ones, but ignore
  imported packages.
- - if there is a `name` macro in the root project, return it
- - if that does not exist but there is a `name` macro in the 'dbt'
-  internal project (or a plugin), return that
- - if neither of those exist (unit tests?), return None
+ - if there is a `generate_{component}_name` macro in the root
+  project, return it
+ - return the `generate_{component}_name` macro from the 'dbt'
+  internal project
  """
  def filter(candidate: MacroCandidate) -> bool:
  return candidate.locality != Locality.Imported
 
  candidates: CandidateList = self._find_macros_by_name(
- name=name,
+ name=f'generate_{component}_name',
  root_project_name=root_project_name,
  # filter out imported packages
  filter=filter,

core/dbt/include/global_project/macros/etc/get_custom_database.sql

@@ -0,0 +1,28 @@
+{#
+ Renders a database name given a custom database name. If the custom
+ database name is none, then the resulting database is just the "database"
+ value in the specified target. If a database override is specified, then
+ the resulting database is the default database concatenated with the
+ custom database.
+
+ This macro can be overriden in projects to define different semantics
+ for rendering a database name.
+
+ Arguments:
+ custom_database_name: The custom database name specified for a model, or none
+ node: The node the database is being generated for
+
+#}
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+ {%- set default_database = target.database -%}
+ {%- if custom_database_name is none -%}
+
+ {{ default_database }}
+
+ {%- else -%}
+
+ {{ custom_database_name }}
+
+ {%- endif -%}
+
+{%- endmacro %}

core/dbt/parser/base.py

@@ -2,7 +2,7 @@
 import itertools
 import os
 from typing import (
- List, Dict, Any, Callable, Iterable, Optional, Generic, TypeVar
+ List, Dict, Any, Iterable, Generic, TypeVar
 )
 
 from hologram import ValidationError
@@ -11,18 +11,17 @@
 from dbt.clients.system import load_file_contents
 from dbt.context.providers import generate_parser_model, generate_parser_macro
 import dbt.flags
-from dbt import deprecations
 from dbt import hooks
 from dbt.adapters.factory import get_adapter
 from dbt.clients.jinja import get_rendered
 from dbt.config import Project, RuntimeConfig
 from dbt.contracts.graph.manifest import (
  Manifest, SourceFile, FilePath, FileHash
 )
-from dbt.contracts.graph.parsed import HasUniqueID, ParsedMacro
+from dbt.contracts.graph.parsed import HasUniqueID
 from dbt.contracts.graph.unparsed import UnparsedNode
 from dbt.exceptions import (
- CompilationException, validator_error_message
+ CompilationException, validator_error_message, InternalException
 )
 from dbt.node_types import NodeType
 from dbt.source_config import SourceConfig
@@ -39,7 +38,6 @@
 FinalNode = TypeVar('FinalNode', bound=ManifestNodes)
 
 
-RelationUpdate = Callable[[Optional[str], IntermediateNode], str]
 ConfiguredBlockType = TypeVar('ConfiguredBlockType', bound=FileBlock)
 
 
@@ -94,6 +92,31 @@ def __init__(
  self.macro_manifest = macro_manifest
 
 
+class RelationUpdate:
+ def __init__(
+ self, config: RuntimeConfig, manifest: Manifest, component: str
+ ) -> None:
+ macro = manifest.find_generate_macro_by_name(
+ component=component,
+ root_project_name=config.project_name,
+ )
+ if macro is None:
+ raise InternalException(
+ f'No macro with name generate_{component}_name found'
+ )
+
+ root_context = generate_parser_macro(macro, config, manifest, None)
+ self.updater = MacroGenerator(macro, root_context)
+ self.component = component
+
+ def __call__(
+ self, parsed_node: Any, config_dict: Dict[str, Any]
+ ) -> None:
+ override = config_dict.get(self.component)
+ new_value = self.updater(override, parsed_node).strip()
+ setattr(parsed_node, self.component, new_value)
+
+
 class ConfiguredParser(
  Parser[FinalNode],
  Generic[ConfiguredBlockType, IntermediateNode, FinalNode],
@@ -106,8 +129,16 @@ def __init__(
  macro_manifest: Manifest,
  ) -> None:
  super().__init__(results, project, root_project, macro_manifest)
- self._get_schema_func: Optional[RelationUpdate] = None
- self._get_alias_func: Optional[RelationUpdate] = None
+
+ self._update_node_database = RelationUpdate(
+ manifest=macro_manifest, config=root_project, component='database'
+ )
+ self._update_node_schema = RelationUpdate(
+ manifest=macro_manifest, config=root_project, component='schema'
+ )
+ self._update_node_alias = RelationUpdate(
+ manifest=macro_manifest, config=root_project, component='alias'
+ )
 
  @abc.abstractclassmethod
  def get_compiled_path(cls, block: ConfiguredBlockType) -> str:
@@ -129,69 +160,6 @@ def default_schema(self):
  def default_database(self):
  return self.root_project.credentials.database
 
- def _build_generate_macro_function(self, macro: ParsedMacro) -> Callable:
- root_context = generate_parser_macro(
- macro, self.root_project, self.macro_manifest, None
- )
- return MacroGenerator(macro, root_context)
-
- def get_schema_func(self) -> RelationUpdate:
- """The get_schema function is set by a few different things:
- - if there is a 'generate_schema_name' macro in the root project,
- it will be used.
- - if that does not exist but there is a 'generate_schema_name'
- macro in the 'dbt' internal project, that will be used
- - if neither of those exist (unit tests?), a function that returns
- the 'default schema' as set in the root project's 'credentials'
- is used
- """
- if self._get_schema_func is not None:
- return self._get_schema_func
-
- get_schema_macro = self.macro_manifest.find_generate_macro_by_name(
- name='generate_schema_name',
- root_project_name=self.root_project.project_name,
- )
- # this is only true in tests!
- if get_schema_macro is None:
- def get_schema(custom_schema_name=None, node=None):
- return self.default_schema
- else:
- get_schema = self._build_generate_macro_function(get_schema_macro)
-
- self._get_schema_func = get_schema
- return self._get_schema_func
-
- def get_alias_func(self) -> RelationUpdate:
- """The get_alias function is set by a few different things:
- - if there is a 'generate_alias_name' macro in the root project,
- it will be used.
- - if that does not exist but there is a 'generate_alias_name'
- macro in the 'dbt' internal project, that will be used
- - if neither of those exist (unit tests?), a function that returns
- the 'default alias' as set in the model's filename or alias
- configuration.
- """
- if self._get_alias_func is not None:
- return self._get_alias_func
-
- get_alias_macro = self.macro_manifest.find_generate_macro_by_name(
- name='generate_alias_name',
- root_project_name=self.root_project.project_name,
- )
- # the generate_alias_name macro might not exist
- if get_alias_macro is None:
- def get_alias(custom_alias_name, node):
- if custom_alias_name is None:
- return node.name
- else:
- return custom_alias_name
- else:
- get_alias = self._build_generate_macro_function(get_alias_macro)
-
- self._get_alias_func = get_alias
- return self._get_alias_func
-
  def get_fqn(self, path: str, name: str) -> List[str]:
  """Get the FQN for the node. This impacts node selection and config
  application.
@@ -297,33 +265,6 @@ def render_with_context(
  parsed_node.raw_sql, context, parsed_node, capture_macros=True
  )
 
- def update_parsed_node_schema(
- self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
- ) -> None:
- # Special macro defined in the global project. Use the root project's
- # definition, not the current package
- schema_override = config_dict.get('schema')
- get_schema = self.get_schema_func()
- try:
- schema = get_schema(schema_override, parsed_node)
- except dbt.exceptions.CompilationException as exc:
- too_many_args = (
- "macro 'dbt_macro__generate_schema_name' takes not more than "
- "1 argument(s)"
- )
- if too_many_args not in str(exc):
- raise
- deprecations.warn('generate-schema-name-single-arg')
- schema = get_schema(schema_override) # type: ignore
- parsed_node.schema = schema.strip()
-
- def update_parsed_node_alias(
- self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
- ) -> None:
- alias_override = config_dict.get('alias')
- get_alias = self.get_alias_func()
- parsed_node.alias = get_alias(alias_override, parsed_node).strip()
-
  def update_parsed_node_config(
  self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
  ) -> None:
@@ -334,6 +275,13 @@ def update_parsed_node_config(
  self._mangle_hooks(final_config_dict)
  parsed_node.config = parsed_node.config.from_dict(final_config_dict)
 
+ def update_parsed_node_name(
+ self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
+ ) -> None:
+ self._update_node_database(parsed_node, config_dict)
+ self._update_node_schema(parsed_node, config_dict)
+ self._update_node_alias(parsed_node, config_dict)
+
  def update_parsed_node(
  self, parsed_node: IntermediateNode, config: SourceConfig
  ) -> None:
@@ -347,15 +295,10 @@ def update_parsed_node(
  model_tags = config_dict.get('tags', [])
  parsed_node.tags.extend(model_tags)
 
- # do this once before we parse the node schema/alias, so
+ # do this once before we parse the node database/schema/alias, so
  # parsed_node.config is what it would be if they did nothing
  self.update_parsed_node_config(parsed_node, config_dict)
-
- parsed_node.database = config_dict.get(
- 'database', self.default_database
- ).strip()
- self.update_parsed_node_schema(parsed_node, config_dict)
- self.update_parsed_node_alias(parsed_node, config_dict)
+ self.update_parsed_node_name(parsed_node, config_dict)
 
  # at this point, we've collected our hooks. Use the node context to
  # render each hook and collect refs/sources

test/integration/006_simple_dependency_test/local_dependency/macros/generate_schema_name.sql

@@ -1,4 +1,15 @@
 {# This should be ignored as it's in a subpackage #}
-{% macro generate_schema_name(custom_schema_name=none) -%}
- invalid_schema_name
+{% macro generate_schema_name(custom_schema_name=none, node=none) -%}
+ {{ exceptions.raise_compiler_error('invalid', node=node) }}
+{%- endmacro %}
+
+{# This should be ignored as it's in a subpackage #}
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+ {{ exceptions.raise_compiler_error('invalid', node=node) }}
+{%- endmacro %}
+
+
+{# This should be ignored as it's in a subpackage #}
+{% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+ {{ exceptions.raise_compiler_error('invalid', node=node) }}
 {%- endmacro %}

test/integration/012_deprecation_tests/test_deprecations.py

@@ -35,30 +35,6 @@ def test_postgres_deprecations(self):
  self.assertEqual(expected, deprecations.active_deprecations)
 
 
-class TestMacroDeprecations(BaseTestDeprecations):
- @property
- def models(self):
- return self.dir('boring-models')
-
- @property
- def project_config(self):
- return {
- 'macro-paths': [self.dir('deprecated-macros')],
- }
-
- @use_profile('postgres')
- def test_postgres_deprecations_fail(self):
- with self.assertRaises(dbt.exceptions.CompilationException):
- self.run_dbt(strict=True)
-
- @use_profile('postgres')
- def test_postgres_deprecations(self):
- self.assertEqual(deprecations.active_deprecations, set())
- self.run_dbt(strict=False)
- expected = {'generate-schema-name-single-arg'}
- self.assertEqual(expected, deprecations.active_deprecations)
-
-
 class TestMaterializationReturnDeprecation(BaseTestDeprecations):
  @property
  def models(self):

test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql

@@ -0,0 +1,10 @@
+
+{% macro generate_database_name(database_name, node) %}
+ {% if database_name == 'alt' %}
+ {{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }}
+ {% elif database_name %}
+ {{ database_name }}
+ {% else %}
+ {{ target.database }}
+ {% endif %}
+{% endmacro %}

test/integration/024_custom_schema_test/db-models/view_1.sql

@@ -0,0 +1,3 @@
+
+
+select * from {{ target.schema }}.seed

test/integration/024_custom_schema_test/db-models/view_2.sql

@@ -0,0 +1,2 @@
+{{ config(database='alt') }}
+select * from {{ ref('view_1') }}