[#3885] Partially parse when environment variables in schema files change

CHANGELOG.md

@@ -33,6 +33,7 @@ Contributors:
 - Prefer macros defined in the project over the ones in a package by default ([#4106](https:/dbt-labs/dbt-core/issues/4106), [#4114](https:/dbt-labs/dbt-core/pull/4114)) 
 - Dependency updates ([#4079](https:/dbt-labs/dbt-core/pull/4079)), ([#3532](https:/dbt-labs/dbt-core/pull/3532)
 - Schedule partial parsing for SQL files with env_var changes ([#3885](https:/dbt-labs/dbt-core/issues/3885), [#4101](https:/dbt-labs/dbt-core/pull/4101))
+- Schedule partial parsing for schema files with env_var changes ([#3885](https:/dbt-labs/dbt-core/issues/3885), [#4162](https:/dbt-labs/dbt-core/pull/4162))
 
 Contributors:
 - [@sungchun12](https:/sungchun12) ([#4017](https:/dbt-labs/dbt/pull/4017))

core/dbt/clients/jinja.py

@@ -21,7 +21,7 @@
 
 from dbt.utils import (
  get_dbt_macro_name, get_docs_macro_name, get_materialization_macro_name,
- get_test_macro_name, deep_map
+ get_test_macro_name, deep_map_render
 )
 
 from dbt.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
@@ -661,5 +661,7 @@ def _convert_function(
 
  return value
 
- kwargs = deep_map(_convert_function, node.test_metadata.kwargs)
+ # The test_metadata.kwargs come from the test builder, and were set
+ # when the test node was created in _parse_generic_test.
+ kwargs = deep_map_render(_convert_function, node.test_metadata.kwargs)
  context[GENERIC_TEST_KWARGS_NAME] = kwargs

core/dbt/config/renderer.py

@@ -5,8 +5,7 @@
 from dbt.exceptions import (
  DbtProjectError, CompilationException, RecursionException
 )
-from dbt.node_types import NodeType
-from dbt.utils import deep_map
+from dbt.utils import deep_map_render
 
 
 Keypath = Tuple[Union[str, int], ...]
@@ -47,7 +46,7 @@ def render_data(
  self, data: Dict[str, Any]
  ) -> Dict[str, Any]:
  try:
- return deep_map(self.render_entry, data)
+ return deep_map_render(self.render_entry, data)
  except RecursionException:
  raise DbtProjectError(
  f'Cycle detected: {self.name} input has a reference to itself',
@@ -163,69 +162,6 @@ def name(self):
  'Profile'
 
 
-class SchemaYamlRenderer(BaseRenderer):
- DOCUMENTABLE_NODES = frozenset(
- n.pluralize() for n in NodeType.documentable()
- )
-
- @property
- def name(self):
- return 'Rendering yaml'
-
- def _is_norender_key(self, keypath: Keypath) -> bool:
- """
- models:
- - name: blah
- - description: blah
- tests: ...
- - columns:
- - name:
- - description: blah
- tests: ...
-
- Return True if it's tests or description - those aren't rendered
- """
- if len(keypath) >= 2 and keypath[1] in ('tests', 'description'):
- return True
-
- if (
- len(keypath) >= 4 and
- keypath[1] == 'columns' and
- keypath[3] in ('tests', 'description')
- ):
- return True
-
- return False
-
- # don't render descriptions or test keyword arguments
- def should_render_keypath(self, keypath: Keypath) -> bool:
- if len(keypath) < 2:
- return True
-
- if keypath[0] not in self.DOCUMENTABLE_NODES:
- return True
-
- if len(keypath) < 3:
- return True
-
- if keypath[0] == NodeType.Source.pluralize():
- if keypath[2] == 'description':
- return False
- if keypath[2] == 'tables':
- if self._is_norender_key(keypath[3:]):
- return False
- elif keypath[0] == NodeType.Macro.pluralize():
- if keypath[2] == 'arguments':
- if self._is_norender_key(keypath[3:]):
- return False
- elif self._is_norender_key(keypath[1:]):
- return False
- else: # keypath[0] in self.DOCUMENTABLE_NODES:
- if self._is_norender_key(keypath[1:]):
- return False
- return True
-
-
 class PackageRenderer(BaseRenderer):
  @property
  def name(self):

core/dbt/context/base.py

@@ -6,12 +6,12 @@
 
 from dbt import flags
 from dbt import tracking
-from dbt.clients.jinja import undefined_error, get_rendered
+from dbt.clients.jinja import get_rendered
 from dbt.clients.yaml_helper import ( # noqa: F401
  yaml, safe_load, SafeLoader, Loader, Dumper
 )
 from dbt.contracts.graph.compiled import CompiledResource
-from dbt.exceptions import raise_compiler_error, MacroReturn
+from dbt.exceptions import raise_compiler_error, MacroReturn, raise_parsing_error
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.version import __version__ as dbt_version
 
@@ -21,6 +21,38 @@
 import datetime
 import re
 
+# Contexts in dbt Core
+# Contexts are used for Jinja rendering. They include context methods,
+# executable macros, and various settings that are available in Jinja.
+#
+# Different contexts are used in different places because we allow access
+# to different methods and data in different places. Executable SQL, for
+# example, includes the available macros and the model, while Jinja in
+# yaml files is more limited.
+#
+# The context that is passed to Jinja is always in a dictionary format,
+# not an actual class, so a 'to_dict()' is executed on a context class
+# before it is used for rendering.
+#
+# Each context has a generate_<name>_context function to create the context.
+# ProviderContext subclasses have different generate functions for
+# parsing and for execution.
+#
+# Context class hierarchy
+#
+# BaseContext -- core/dbt/context/base.py
+# TargetContext -- core/dbt/context/target.py
+# ConfiguredContext -- core/dbt/context/configured.py
+# SchemaYamlContext -- core/dbt/context/configured.py
+# DocsRuntimeContext -- core/dbt/context/configured.py
+# MacroResolvingContext -- core/dbt/context/configured.py
+# ManifestContext -- core/dbt/context/manifest.py
+# QueryHeaderContext -- core/dbt/context/manifest.py
+# ProviderContext -- core/dbt/context/provider.py
+# MacroContext -- core/dbt/context/provider.py
+# ModelContext -- core/dbt/context/provider.py
+# TestContext -- core/dbt/context/provider.py
+
 
 def get_pytz_module_context() -> Dict[str, Any]:
  context_exports = pytz.__all__ # type: ignore
@@ -164,8 +196,6 @@ class BaseContext(metaclass=ContextMeta):
  def __init__(self, cli_vars):
  self._ctx = {}
  self.cli_vars = cli_vars
- # Save the env_vars encountered using this
- self.env_vars = {}
 
  def generate_builtins(self):
  builtins: Dict[str, Any] = {}
@@ -287,11 +317,10 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  return_value = default
 
  if return_value is not None:
- self.env_vars[var] = return_value
  return return_value
  else:
  msg = f"Env var required but not provided: '{var}'"
- undefined_error(msg)
+ raise_parsing_error(msg)
 
  if os.environ.get('DBT_MACRO_DEBUGGING'):
  @contextmember

core/dbt/context/configured.py

@@ -1,11 +1,14 @@
-from typing import Any, Dict
+import os
+from typing import Any, Dict, Optional
 
 from dbt.contracts.connection import AdapterRequiredConfig
+from dbt.logger import SECRET_ENV_PREFIX
 from dbt.node_types import NodeType
 from dbt.utils import MultiDict
 
-from dbt.context.base import contextproperty, Var
+from dbt.context.base import contextproperty, contextmember, Var
 from dbt.context.target import TargetContext
+from dbt.exceptions import raise_parsing_error
 
 
 class ConfiguredContext(TargetContext):
@@ -64,18 +67,41 @@ def __call__(self, var_name, default=Var._VAR_NOTSET):
  return self.get_missing_var(var_name)
 
 
+class SchemaYamlVars():
+ def __init__(self):
+ self.env_vars = {}
+ self.vars = {}
+
+
 class SchemaYamlContext(ConfiguredContext):
  # subclass is DocsRuntimeContext
- def __init__(self, config, project_name: str):
+ def __init__(self, config, project_name: str, schema_yaml_vars: Optional[SchemaYamlVars]):
  super().__init__(config)
  self._project_name = project_name
+ self.schema_yaml_vars = schema_yaml_vars
 
  @contextproperty
  def var(self) -> ConfiguredVar:
  return ConfiguredVar(
  self._ctx, self.config, self._project_name
  )
 
+ @contextmember
+ def env_var(self, var: str, default: Optional[str] = None) -> str:
+ return_value = None
+ if var in os.environ:
+ return_value = os.environ[var]
+ elif default is not None:
+ return_value = default
+
+ if return_value is not None:
+ if not var.startswith(SECRET_ENV_PREFIX) and self.schema_yaml_vars:
+ self.schema_yaml_vars.env_vars[var] = return_value
+ return return_value
+ else:
+ msg = f"Env var required but not provided: '{var}'"
+ raise_parsing_error(msg)
+
 
 class MacroResolvingContext(ConfiguredContext):
  def __init__(self, config):
@@ -89,9 +115,9 @@ def var(self) -> ConfiguredVar:
 
 
 def generate_schema_yml_context(
- config: AdapterRequiredConfig, project_name: str
+  config: AdapterRequiredConfig, project_name: str, schema_yaml_vars: SchemaYamlVars = None
 ) -> Dict[str, Any]:
- ctx = SchemaYamlContext(config, project_name)
+ ctx = SchemaYamlContext(config, project_name, schema_yaml_vars)
  return ctx.to_dict()
 
 

core/dbt/context/docs.py

@@ -23,7 +23,7 @@ def __init__(
  manifest: Manifest,
  current_project: str,
  ) -> None:
- super().__init__(config, current_project)
+ super().__init__(config, current_project, None)
  self.node = node
  self.manifest = manifest
 

core/dbt/context/providers.py

@@ -11,11 +11,12 @@
  get_adapter, get_adapter_package_names, get_adapter_type_names
 )
 from dbt.clients import agate_helper
-from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack, undefined_error
+from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack
 from dbt.config import RuntimeConfig, Project
 from .base import contextmember, contextproperty, Var
 from .configured import FQNLookup
 from .context_config import ContextConfig
+from dbt.logger import SECRET_ENV_PREFIX
 from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
 from .macros import MacroNamespaceBuilder, MacroNamespace
 from .manifest import ManifestContext
@@ -47,6 +48,7 @@
  ref_bad_context,
  source_target_not_found,
  wrapped_exports,
+ raise_parsing_error,
 )
 from dbt.config import IsFQNResource
 from dbt.logger import GLOBAL_LOGGER as logger, SECRET_ENV_PREFIX # noqa
@@ -1176,15 +1178,19 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
  return_value = default
 
  if return_value is not None:
- # Save the env_var value in the manifest and the var name in the source_file
- if not var.startswith(SECRET_ENV_PREFIX) and self.model:
+ # Save the env_var value in the manifest and the var name in the source_file.
+ # If this is compiling, do not save because it's irrelevant to parsing.
+ if (not var.startswith(SECRET_ENV_PREFIX) and self.model and
+ not hasattr(self.model, 'compiled')):
  self.manifest.env_vars[var] = return_value
  source_file = self.manifest.files[self.model.file_id]
- source_file.env_vars.append(var)
+ # Schema files should never get here
+ if source_file.parse_file_type != 'schema':
+ source_file.env_vars.append(var)
  return return_value
  else:
  msg = f"Env var required but not provided: '{var}'"
- undefined_error(msg)
+ raise_parsing_error(msg)
 
 
 class MacroContext(ProviderContext):
@@ -1428,6 +1434,28 @@ def _build_test_namespace(self):
  )
  self.namespace = macro_namespace
 
+ @contextmember
+ def env_var(self, var: str, default: Optional[str] = None) -> str:
+ return_value = None
+ if var in os.environ:
+ return_value = os.environ[var]
+ elif default is not None:
+ return_value = default
+
+ if return_value is not None:
+ # Save the env_var value in the manifest and the var name in the source_file
+ if not var.startswith(SECRET_ENV_PREFIX) and self.model:
+ self.manifest.env_vars[var] = return_value
+ # the "model" should only be test nodes, but just in case, check
+ if self.model.resource_type == NodeType.Test and self.model.file_key_name:
+ source_file = self.manifest.files[self.model.file_id]
+ (yaml_key, name) = self.model.file_key_name.split('.')
+ source_file.add_env_var(var, yaml_key, name)
+ return return_value
+ else:
+ msg = f"Env var required but not provided: '{var}'"
+ raise_parsing_error(msg)
+
 
 def generate_test_context(
  model: ManifestNode,

core/dbt/contracts/files.py

@@ -301,5 +301,21 @@ def get_all_test_ids(self):
  test_ids.extend(self.tests[key][name])
  return test_ids
 
+ def add_env_var(self, var, yaml_key, name):
+ if yaml_key not in self.env_vars:
+ self.env_vars[yaml_key] = {}
+ if name not in self.env_vars[yaml_key]:
+ self.env_vars[yaml_key][name] = []
+ if var not in self.env_vars[yaml_key][name]:
+ self.env_vars[yaml_key][name].append(var)
+
+ def delete_from_env_vars(self, yaml_key, name):
+ # We delete all vars for this yaml_key/name because the
+ # entry has been scheduled for reparsing.
+ if yaml_key in self.env_vars and name in self.env_vars[yaml_key]:
+ del self.env_vars[yaml_key][name]
+ if not self.env_vars[yaml_key]:
+ del self.env_vars[yaml_key]
+
 
 AnySourceFile = Union[SchemaSourceFile, SourceFile]

core/dbt/contracts/graph/compiled.py

@@ -126,6 +126,7 @@ class CompiledGenericTestNode(CompiledNode, HasTestMetadata):
  # keep this in sync with ParsedGenericTestNode!
  resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
  column_name: Optional[str] = None
+ file_key_name: Optional[str] = None
  # Was not able to make mypy happy and keep the code working. We need to
  # refactor the various configs.
  config: TestConfig = field(default_factory=TestConfig) # type:ignore

core/dbt/contracts/graph/parsed.py

@@ -410,6 +410,9 @@ def same_body(self: T, other: T) -> bool:
 @dataclass
 class TestMetadata(dbtClassMixin, Replaceable):
  name: str
+ # kwargs are the args that are left in the test builder after
+ # removing configs. They are set from the test builder when
+ # the test node is created.
  kwargs: Dict[str, Any] = field(default_factory=dict)
  namespace: Optional[str] = None
 
@@ -432,6 +435,7 @@ class ParsedGenericTestNode(ParsedNode, HasTestMetadata):
  # keep this in sync with CompiledGenericTestNode!
  resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
  column_name: Optional[str] = None
+ file_key_name: Optional[str] = None
  # Was not able to make mypy happy and keep the code working. We need to
  # refactor the various configs.
  config: TestConfig = field(default_factory=TestConfig) # type: ignore