create a decorator for stubbing out methods at parse time

Includes some unit tests

core/dbt/adapters/base/impl.py

@@ -17,7 +17,8 @@
 from dbt.schema import Column
 from dbt.utils import filter_null_values
 
-from dbt.adapters.base.meta import AdapterMeta, available, available_deprecated
+
+from dbt.adapters.base.meta import AdapterMeta, available
 from dbt.adapters.base import BaseRelation
 from dbt.adapters.cache import RelationsCache
 
@@ -221,7 +222,7 @@ def connection_named(self, name):
  finally:
  self.release_connection()
 
- @available
+ @available.parse(lambda *a, **k: ('', dbt.clients.agate_helper()))
  def execute(self, sql, auto_begin=False, fetch=False):
  """Execute the given SQL. This is a thin wrapper around
  ConnectionManager.execute.
@@ -404,7 +405,7 @@ def check_schema_exists(self, database, schema):
  # Abstract methods about relations
  ###
  @abc.abstractmethod
- @available
+ @available.parse_none
  def drop_relation(self, relation):
  """Drop the given relation.
 
@@ -417,7 +418,7 @@ def drop_relation(self, relation):
  )
 
  @abc.abstractmethod
- @available
+ @available.parse_none
  def truncate_relation(self, relation):
  """Truncate the given relation.
 
@@ -428,7 +429,7 @@ def truncate_relation(self, relation):
  )
 
  @abc.abstractmethod
- @available
+ @available.parse_none
  def rename_relation(self, from_relation, to_relation):
  """Rename the relation from from_relation to to_relation.
 
@@ -442,7 +443,7 @@ def rename_relation(self, from_relation, to_relation):
  )
 
  @abc.abstractmethod
- @available
+ @available.parse_list
  def get_columns_in_relation(self, relation):
  """Get a list of the columns in the given Relation.
 
@@ -454,7 +455,7 @@ def get_columns_in_relation(self, relation):
  '`get_columns_in_relation` is not implemented for this adapter!'
  )
 
- @available_deprecated('get_columns_in_relation')
+ @available.deprecated('get_columns_in_relation', lambda *a, **k: [])
  def get_columns_in_table(self, schema, identifier):
  """DEPRECATED: Get a list of the columns in the given table."""
  relation = self.Relation.create(
@@ -488,7 +489,7 @@ def list_relations_without_caching(self, information_schema, schema):
  relations from.
  :param str schema: The name of the schema to list relations from.
  :return: The relations in schema
- :retype: List[self.Relation]
+ :rtype: List[self.Relation]
  """
  raise dbt.exceptions.NotImplementedException(
  '`list_relations_without_caching` is not implemented for this '
@@ -498,10 +499,17 @@ def list_relations_without_caching(self, information_schema, schema):
  ###
  # Provided methods about relations
  ###
- @available
+ @available.parse_list
  def get_missing_columns(self, from_relation, to_relation):
- """Returns dict of {column:type} for columns in from_table that are
- missing from to_relation
+ """Returns a list of Columns in from_relation that are missing from
+ to_relation.
+
+ :param Relation from_relation: The relation that might have extra
+ columns
+ :param Relation to_relation: The realtion that might have columns
+ missing
+ :return: The columns in from_relation that are missing from to_relation
+ :rtype: List[self.Relation]
  """
  if not isinstance(from_relation, self.Relation):
  dbt.exceptions.invalid_type_error(
@@ -534,7 +542,7 @@ def get_missing_columns(self, from_relation, to_relation):
  if col_name in missing_columns
  ]
 
- @available
+ @available.parse_none
  def expand_target_column_types(self, temp_table, to_relation):
  if not isinstance(to_relation, self.Relation):
  dbt.exceptions.invalid_type_error(
@@ -600,7 +608,7 @@ def _make_match(self, relations_list, database, schema, identifier):
 
  return matches
 
- @available
+ @available.parse_none
  def get_relation(self, database, schema, identifier):
  relations_list = self.list_relations(database, schema)
 
@@ -622,7 +630,7 @@ def get_relation(self, database, schema, identifier):
 
  return None
 
- @available_deprecated('get_relation')
+ @available.deprecated('get_relation', lambda *a, **k: False)
  def already_exists(self, schema, name):
  """DEPRECATED: Return if a model already exists in the database"""
  database = self.config.credentials.database
@@ -634,7 +642,7 @@ def already_exists(self, schema, name):
  # although some adapters may override them
  ###
  @abc.abstractmethod
- @available
+ @available.parse_none
  def create_schema(self, database, schema):
  """Create the given schema if it does not exist.
 

core/dbt/adapters/base/meta.py

@@ -3,26 +3,42 @@
 from dbt.deprecations import warn, renamed_method
 
 
+def _always_none(*args, **kwargs):
+ return None
+
+
+def _always_list(*args, **kwargs):
+ return None
+
+
 def available(func):
- """A decorator to indicate that a method on the adapter will be exposed to
- the database wrapper, and the model name will be injected into the
- arguments.
+ """A decorator to indicate that a method on the adapter will be
+ exposed to the database wrapper, and will be available at parse and run
+ time.
  """
  func._is_available_ = True
  return func
 
 
-def available_deprecated(supported_name):
+def available_deprecated(supported_name, parse_replacement=None):
  """A decorator that marks a function as available, but also prints a
  deprecation warning. Use like
 
  @available_deprecated('my_new_method')
- def my_old_method(self, arg, model_name=None):
+ def my_old_method(self, arg):
  args = compatability_shim(arg)
- return self.my_new_method(*args, model_name=None)
+ return self.my_new_method(*args)
+
+ @available_deprecated('my_new_slow_method', lambda *a, **k: (0, ''))
+ def my_old_slow_method(self, arg):
+ args = compatibility_shim(arg)
+ return self.my_new_slow_method(*args)
 
  To make `adapter.my_old_method` available but also print out a warning on
  use directing users to `my_new_method`.
+
+ The optional parse_replacement, if provided, will provide a parse-time
+ replacement for the actual method (see `available_parse`).
  """
  def wrapper(func):
  func_name = func.__name__
@@ -32,10 +48,43 @@ def wrapper(func):
  def inner(*args, **kwargs):
  warn('adapter:{}'.format(func_name))
  return func(*args, **kwargs)
+
+ if parse_replacement:
+ available = available_parse(parse_replacement)
  return available(inner)
  return wrapper
 
 
+def available_parse(parse_replacement):
+ """A decorator factory to indicate that a method on the adapter will be
+ exposed to the database wrapper, and will be stubbed out at parse time with
+ the given function.
+
+ @available_parse()
+ def my_method(self, a, b):
+ if something:
+ return None
+ return big_expensive_db_query()
+
+ @available_parse(lambda *args, **args: {})
+ def my_other_method(self, a, b):
+ x = {}
+ x.update(big_expensive_db_query())
+ return x
+ """
+ def inner(func):
+ func._parse_replacement_ = parse_replacement
+ available(func)
+ return func
+ return inner
+
+
+available.deprecated = available_deprecated
+available.parse = available_parse
+available.parse_none = available_parse(lambda *a, **k: None)
+available.parse_list = available_parse(lambda *a, **k: [])
+
+
 class AdapterMeta(abc.ABCMeta):
  def __new__(mcls, name, bases, namespace, **kwargs):
  cls = super(AdapterMeta, mcls).__new__(mcls, name, bases, namespace,
@@ -47,15 +96,22 @@ def __new__(mcls, name, bases, namespace, **kwargs):
  # injected into the arguments. All methods in here are exposed to the
  # context.
  available = set()
+ replacements = {}
 
  # collect base class data first
  for base in bases:
  available.update(getattr(base, '_available_', set()))
+ replacements.update(getattr(base, '_parse_replacements_', set()))
 
  # override with local data if it exists
  for name, value in namespace.items():
  if getattr(value, '_is_available_', False):
  available.add(name)
+ parse_replacement = getattr(value, '_parse_replacement_', None)
+ if parse_replacement is not None:
+ replacements[name] = parse_replacement
 
  cls._available_ = frozenset(available)
+ # should this be a namedtuple so it will be immutable like _available_?
+ cls._parse_replacements_ = replacements
  return cls

core/dbt/adapters/sql/impl.py

@@ -35,7 +35,7 @@ class SQLAdapter(BaseAdapter):
  - list_relations_without_caching
  - get_columns_in_relation
  """
- @available
+ @available.parse(lambda *a, **k: (None, None))
  def add_query(self, sql, auto_begin=True, bindings=None,
  abridge_sql_log=False):
  """Add a query to the current transaction. A thin wrapper around

core/dbt/context/common.py

@@ -47,7 +47,7 @@ def create(self, *args, **kwargs):
  return self.relation_type.create(*args, **kwargs)
 
 
-class DatabaseWrapper(object):
+class BaseDatabaseWrapper(object):
  """
  Wrapper for runtime database interaction. Applies the runtime quote policy
  via a relation proxy.
@@ -57,14 +57,7 @@ def __init__(self, adapter):
  self.Relation = RelationProxy(adapter)
 
  def __getattr__(self, name):
- if name in self.adapter._available_:
- return getattr(self.adapter, name)
- else:
- raise AttributeError(
- "'{}' object has no attribute '{}'".format(
- self.__class__.__name__, name
- )
- )
+ raise NotImplementedError('subclasses need to implement this')
 
  @property
  def config(self):
@@ -371,7 +364,7 @@ def generate_base(model, model_dict, config, manifest, source_config,
  pre_hooks = None
  post_hooks = None
 
- db_wrapper = DatabaseWrapper(adapter)
+ db_wrapper = provider.DatabaseWrapper(adapter)
 
  context = dbt.utils.merge(context, {
  "adapter": db_wrapper,

core/dbt/context/parser.py

@@ -1,6 +1,7 @@
 import dbt.exceptions
 
 import dbt.context.common
+from dbt.clients.agate_helper import empty_table
 from dbt.adapters.factory import get_adapter
 
 
@@ -97,6 +98,26 @@ def get(self, name, validator=None, default=None):
  return ''
 
 
+class DatabaseWrapper(dbt.context.common.BaseDatabaseWrapper):
+ """The parser subclass of the database wrapper applies any explicit
+ parse-time overrides.
+ """
+ def __getattr__(self, name):
+ override = (name in self.adapter._available_ and
+ name in self.adapter._parse_replacements_)
+
+ if override:
+ return self.adapter._parse_replacements_[name]
+ elif name in self.adapter._available_:
+ return getattr(self.adapter, name)
+ else:
+ raise AttributeError(
+ "'{}' object has no attribute '{}'".format(
+ self.__class__.__name__, name
+ )
+ )
+
+
 def generate(model, runtime_config, manifest, source_config):
  # during parsing, we don't have a connection, but we might need one, so we
  # have to acquire it.

core/dbt/context/runtime.py

@@ -118,6 +118,21 @@ def get(self, name, validator=None, default=None):
  return to_return
 
 
+class DatabaseWrapper(dbt.context.common.BaseDatabaseWrapper):
+ """The runtime database wrapper exposes everything the adapter marks
+ available.
+ """
+ def __getattr__(self, name):
+ if name in self.adapter._available_:
+ return getattr(self.adapter, name)
+ else:
+ raise AttributeError(
+ "'{}' object has no attribute '{}'".format(
+ self.__class__.__name__, name
+ )
+ )
+
+
 def generate(model, runtime_config, manifest):
  return dbt.context.common.generate(
  model, runtime_config, manifest, None, dbt.context.runtime)