Merge branch 'dev/wilt-chamberlain' into build/pr-fixup-1285

core/dbt/adapters/base/impl.py

@@ -387,6 +387,7 @@ def list_schemas(self, database):
  '`list_schemas` is not implemented for this adapter!'
  )
 
+ @available.parse(lambda *a, **k: False)
  def check_schema_exists(self, database, schema):
  """Check if a schema exists.
 
@@ -542,7 +543,7 @@ def get_missing_columns(self, from_relation, to_relation):
  ]
 
  @available.parse_none
- def valid_archive_target(self, relation):
+ def valid_snapshot_target(self, relation):
  """Ensure that the target relation is valid, by making sure it has the
  expected columns.
 
@@ -552,7 +553,7 @@ def valid_archive_target(self, relation):
  """
  if not isinstance(relation, self.Relation):
  dbt.exceptions.invalid_type_error(
- method_name='is_existing_old_style_archive',
+ method_name='valid_snapshot_target',
  arg_name='relation',
  got_value=relation,
  expected_type=self.Relation)
@@ -572,34 +573,34 @@ def valid_archive_target(self, relation):
  if missing:
  if extra:
  msg = (
- 'Archive target has ("{}") but not ("{}") - is it an '
+ 'Snapshot target has ("{}") but not ("{}") - is it an '
  'unmigrated previous version archive?'
  .format('", "'.join(extra), '", "'.join(missing))
  )
  else:
  msg = (
- 'Archive target is not an archive table (missing "{}")'
+ 'Snapshot target is not a snapshot table (missing "{}")'
  .format('", "'.join(missing))
  )
  dbt.exceptions.raise_compiler_error(msg)
 
  @available.parse_none
- def expand_target_column_types(self, temp_table, to_relation):
+ def expand_target_column_types(self, from_relation, to_relation):
+ if not isinstance(from_relation, self.Relation):
+ dbt.exceptions.invalid_type_error(
+ method_name='expand_target_column_types',
+ arg_name='from_relation',
+ got_value=from_relation,
+ expected_type=self.Relation)
+
  if not isinstance(to_relation, self.Relation):
  dbt.exceptions.invalid_type_error(
  method_name='expand_target_column_types',
  arg_name='to_relation',
  got_value=to_relation,
  expected_type=self.Relation)
 
- goal = self.Relation.create(
- database=None,
- schema=None,
- identifier=temp_table,
- type='table',
- quote_policy=self.config.quoting
- )
- self.expand_column_types(goal, to_relation)
+ self.expand_column_types(from_relation, to_relation)
 
  def list_relations(self, database, schema):
  if self._schema_is_cached(database, schema):

core/dbt/clients/_jinja_blocks.py

@@ -52,6 +52,8 @@ def end_pat(self):
 RAW_START_PATTERN = regex(
  r'(?:\s*\{\%\-|\{\%)\s*(?P<raw_start>(raw))\s*(?:\-\%\}\s*|\%\})'
 )
+EXPR_START_PATTERN = regex(r'(?P<expr_start>(\{\{\s*))')
+EXPR_END_PATTERN = regex(r'(?P<expr_end>(\s*\}\}))')
 
 BLOCK_START_PATTERN = regex(''.join((
  r'(?:\s*\{\%\-|\{\%)\s*',
@@ -92,6 +94,8 @@ def end_pat(self):
  r'"([^"\\]*(?:\\.[^"\\]*)*)"))'
 )
 
+QUOTE_START_PATTERN = regex(r'''(?P<quote>(['"]))''')
+
 # any number of non-quote characters, followed by:
 # - quote: a quote mark indicating start of a string (you'll want to backtrack
 # the regex end on quotes and then match with the string pattern)
@@ -179,6 +183,31 @@ def _expect_match(self, expected_name, *patterns, **kwargs):
  dbt.exceptions.raise_compiler_error(msg)
  return match
 
+ def handle_expr(self):
+ """Handle an expression. At this point we're at a string like:
+ {{ 1 + 2 }}
+ ^ right here
+
+ We expect to find a `}}`, but we might find one in a string before
+ that. Imagine the case of `{{ 2 * "}}" }}`...
+
+ You're not allowed to have blocks or comments inside an expr so it is
+ pretty straightforward, I hope: only strings can get in the way.
+ """
+ while True:
+ match = self._expect_match('}}',
+ EXPR_END_PATTERN,
+ QUOTE_START_PATTERN)
+ if match.groupdict().get('expr_end') is not None:
+ break
+ else:
+ # it's a quote. we haven't advanced for this match yet, so
+ # just slurp up the whole string, no need to rewind.
+ match = self._expect_match('string', STRING_PATTERN)
+ self.advance(match.end())
+
+ self.advance(match.end())
+
  def handle_block(self, match, block_start=None):
  """Handle a block. The current state of the parser should be after the
  open block is completed:
@@ -197,12 +226,18 @@ def handle_block(self, match, block_start=None):
 
  self._block_contents = ''
 
+ search = [found.end_pat(), COMMENT_START_PATTERN, RAW_START_PATTERN,
+ EXPR_START_PATTERN]
+
+ # docs and macros do not honor embedded quotes
+ if found.block_type_name not in ('docs', 'macro'):
+ # is this right?
+ search.append(QUOTE_START_PATTERN)
+
  # you can have as many comments in your block as you'd like!
  while True:
  match = self._expect_match(
- '"{}"'.format(found.end_block_type_name),
- found.end_pat(), COMMENT_START_PATTERN, RAW_START_PATTERN,
- regex('''(?P<quote>(['"]))''')
+ '"{}"'.format(found.end_block_type_name), *search
  )
  groups = match.groupdict()
  if groups.get('endblock') is not None:
@@ -218,6 +253,8 @@ def handle_block(self, match, block_start=None):
  self.rewind()
  match = self._expect_match('any string', STRING_PATTERN)
  self.advance(match.end())
+ elif groups.get('expr_start') is not None:
+ self.handle_expr()
  else:
  raise dbt.exceptions.InternalException(
  'unhandled regex in handle_block, no match: {}'

core/dbt/clients/registry.py

@@ -5,6 +5,7 @@
 from dbt.utils import memoized
 from dbt.logger import GLOBAL_LOGGER as logger
 import os
+import time
 
 if os.getenv('DBT_PACKAGE_HUB_URL'):
  DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
@@ -22,11 +23,20 @@ def _get_url(url, registry_base_url=None):
 def _wrap_exceptions(fn):
  @wraps(fn)
  def wrapper(*args, **kwargs):
- try:
- return fn(*args, **kwargs)
- except requests.exceptions.ConnectionError as e:
- six.raise_from(
- RegistryException('Unable to connect to registry hub'), e)
+ max_attempts = 5
+ attempt = 0
+ while True:
+ attempt += 1
+ try:
+ return fn(*args, **kwargs)
+ except requests.exceptions.ConnectionError as exc:
+ if attempt < max_attempts:
+ time.sleep(1)
+ continue
+ six.raise_from(
+ RegistryException('Unable to connect to registry hub'),
+ exc
+ )
  return wrapper
 
 

core/dbt/compilation.py

@@ -27,7 +27,7 @@ def print_compile_stats(stats):
  names = {
  NodeType.Model: 'models',
  NodeType.Test: 'tests',
- NodeType.Archive: 'archives',
+ NodeType.Snapshot: 'snapshots',
  NodeType.Analysis: 'analyses',
  NodeType.Macro: 'macros',
  NodeType.Operation: 'operations',
@@ -140,9 +140,9 @@ def compile_node(self, node, manifest, extra_context=None):
  # don't wrap schema tests or analyses.
  injected_node.wrapped_sql = injected_node.injected_sql
 
- elif is_type(injected_node, NodeType.Archive):
+ elif is_type(injected_node, NodeType.Snapshot):
  # unfortunately we do everything automagically for
- # archives. in the future it'd be nice to generate
+ # snapshots. in the future it'd be nice to generate
  # the SQL at the parser level.
  pass
 
@@ -209,7 +209,7 @@ def _is_writable(node):
  if not node.injected_sql:
  return False
 
- if dbt.utils.is_type(node, NodeType.Archive):
+ if dbt.utils.is_type(node, NodeType.Snapshot):
  return False
 
  return True

core/dbt/config/profile.py

@@ -11,6 +11,8 @@
 from dbt.exceptions import RuntimeException
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.utils import parse_cli_vars
+from dbt import tracking
+from dbt.ui import printer
 
 from .renderer import ConfigRenderer
 
@@ -95,6 +97,18 @@ def from_directory(cls, directory):
  user_cfg = profile.get('config', {})
  return cls.from_dict(user_cfg)
 
+ def set_values(self, cookie_dir):
+ if self.send_anonymous_usage_stats:
+ tracking.initialize_tracking(cookie_dir)
+ else:
+ tracking.do_not_track()
+
+ if self.use_colors:
+ printer.use_colors()
+
+ if self.printer_width:
+ printer.printer_width(self.printer_width)
+
 
 class Profile(object):
  def __init__(self, profile_name, target_name, config, threads,

core/dbt/config/project.py

@@ -143,7 +143,7 @@ def _parse_versions(versions):
 class Project(object):
  def __init__(self, project_name, version, project_root, profile_name,
  source_paths, macro_paths, data_paths, test_paths,
- analysis_paths, docs_paths, target_path, archive_paths,
+ analysis_paths, docs_paths, target_path, snapshot_paths,
  clean_targets, log_path, modules_path, quoting, models,
  on_run_start, on_run_end, archive, seeds, dbt_version,
  packages):
@@ -158,7 +158,7 @@ def __init__(self, project_name, version, project_root, profile_name,
  self.analysis_paths = analysis_paths
  self.docs_paths = docs_paths
  self.target_path = target_path
- self.archive_paths = archive_paths
+ self.snapshot_paths = snapshot_paths
  self.clean_targets = clean_targets
  self.log_path = log_path
  self.modules_path = modules_path
@@ -241,7 +241,7 @@ def from_project_config(cls, project_dict, packages_dict=None):
  analysis_paths = project_dict.get('analysis-paths', [])
  docs_paths = project_dict.get('docs-paths', source_paths[:])
  target_path = project_dict.get('target-path', 'target')
- archive_paths = project_dict.get('archive-paths', ['archives'])
+ snapshot_paths = project_dict.get('snapshot-paths', ['snapshots'])
  # should this also include the modules path by default?
  clean_targets = project_dict.get('clean-targets', [target_path])
  log_path = project_dict.get('log-path', 'logs')
@@ -275,7 +275,7 @@ def from_project_config(cls, project_dict, packages_dict=None):
  analysis_paths=analysis_paths,
  docs_paths=docs_paths,
  target_path=target_path,
- archive_paths=archive_paths,
+ snapshot_paths=snapshot_paths,
  clean_targets=clean_targets,
  log_path=log_path,
  modules_path=modules_path,
@@ -323,7 +323,7 @@ def to_project_config(self, with_packages=False):
  'analysis-paths': self.analysis_paths,
  'docs-paths': self.docs_paths,
  'target-path': self.target_path,
- 'archive-paths': self.archive_paths,
+ 'snapshot-paths': self.snapshot_paths,
  'clean-targets': self.clean_targets,
  'log-path': self.log_path,
  'quoting': self.quoting,

core/dbt/config/runtime.py

@@ -12,13 +12,22 @@
 from .project import Project
 
 
+_ARCHIVE_REMOVED_MESSAGE = '''
+The `archive` section in `dbt_project.yml` is no longer supported. Please use a
+`snapshot` block instead. For more information on snapshot blocks and a script
+to help migrate these archives, please consult the 0.14.0 migration guide:
+
+https://docs.getdbt.com/v0.14/docs/upgrading-to-014
+'''.strip()
+
+
 class RuntimeConfig(Project, Profile):
  """The runtime configuration, as constructed from its components. There's a
  lot because there is a lot of stuff!
  """
  def __init__(self, project_name, version, project_root, source_paths,
  macro_paths, data_paths, test_paths, analysis_paths,
- docs_paths, target_path, archive_paths, clean_targets,
+ docs_paths, target_path, snapshot_paths, clean_targets,
  log_path, modules_path, quoting, models, on_run_start,
  on_run_end, archive, seeds, dbt_version, profile_name,
  target_name, config, threads, credentials, packages, args):
@@ -39,7 +48,7 @@ def __init__(self, project_name, version, project_root, source_paths,
  analysis_paths=analysis_paths,
  docs_paths=docs_paths,
  target_path=target_path,
- archive_paths=archive_paths,
+ snapshot_paths=snapshot_paths,
  clean_targets=clean_targets,
  log_path=log_path,
  modules_path=modules_path,
@@ -64,19 +73,25 @@ def __init__(self, project_name, version, project_root, source_paths,
  self.validate()
 
  @classmethod
- def from_parts(cls, project, profile, args):
+ def from_parts(cls, project, profile, args, allow_archive_configs=False):
  """Instantiate a RuntimeConfig from its components.
 
  :param profile Profile: A parsed dbt Profile.
  :param project Project: A parsed dbt Project.
  :param args argparse.Namespace: The parsed command-line arguments.
+ :param allow_archive_configs bool: If True, ignore archive blocks in
+ configs. This flag exists to enable archive migration.
  :returns RuntimeConfig: The new configuration.
  """
  quoting = deepcopy(
  get_relation_class_by_name(profile.credentials.type)
  .DEFAULTS['quote_policy']
  )
  quoting.update(project.quoting)
+ if project.archive and not allow_archive_configs:
+ # if the user has an `archive` section, raise an error
+ raise DbtProjectError(_ARCHIVE_REMOVED_MESSAGE)
+
  return cls(
  project_name=project.project_name,
  version=project.version,
@@ -88,7 +103,7 @@ def from_parts(cls, project, profile, args):
  analysis_paths=project.analysis_paths,
  docs_paths=project.docs_paths,
  target_path=project.target_path,
- archive_paths=project.archive_paths,
+ snapshot_paths=project.snapshot_paths,
  clean_targets=project.clean_targets,
  log_path=project.log_path,
  modules_path=project.modules_path,
@@ -163,12 +178,14 @@ def validate(self):
  self.validate_version()
 
  @classmethod
- def from_args(cls, args):
+ def from_args(cls, args, allow_archive_configs=False):
  """Given arguments, read in dbt_project.yml from the current directory,
  read in packages.yml if it exists, and use them to find the profile to
  load.
 
  :param args argparse.Namespace: The arguments as parsed from the cli.
+ :param allow_archive_configs bool: If True, ignore archive blocks in
+ configs. This flag exists to enable archive migration.
  :raises DbtProjectError: If the project is invalid or missing.
  :raises DbtProfileError: If the profile is invalid or missing.
  :raises ValidationException: If the cli variables are invalid.
@@ -185,5 +202,6 @@ def from_args(cls, args):
  return cls.from_parts(
  project=project,
  profile=profile,
- args=args
+ args=args,
+ allow_archive_configs=allow_archive_configs
  )