Merge pull request #1581 from fishtown-analytics/feature/remove-archi…

…ve-config

remove archive block support and "dbt snapshot-migrate" (#1580)

core/dbt/config/project.py

@@ -144,8 +144,7 @@ def __init__(self, project_name, version, project_root, profile_name,
  source_paths, macro_paths, data_paths, test_paths,
  analysis_paths, docs_paths, target_path, snapshot_paths,
  clean_targets, log_path, modules_path, quoting, models,
- on_run_start, on_run_end, archive, seeds, dbt_version,
- packages):
+ on_run_start, on_run_end, seeds, dbt_version, packages):
  self.project_name = project_name
  self.version = version
  self.project_root = project_root
@@ -165,7 +164,6 @@ def __init__(self, project_name, version, project_root, profile_name,
  self.models = models
  self.on_run_start = on_run_start
  self.on_run_end = on_run_end
- self.archive = archive
  self.seeds = seeds
  self.dbt_version = dbt_version
  self.packages = packages
@@ -176,7 +174,6 @@ def _preprocess(project_dict):
  into empty containers, and to turn strings into arrays of strings.
  """
  handlers = {
- ('archive',): _list_if_none,
  ('on-run-start',): _list_if_none_or_string,
  ('on-run-end',): _list_if_none_or_string,
  }
@@ -251,7 +248,6 @@ def from_project_config(cls, project_dict, packages_dict=None):
  models = project_dict.get('models', {})
  on_run_start = project_dict.get('on-run-start', [])
  on_run_end = project_dict.get('on-run-end', [])
- archive = project_dict.get('archive', [])
  seeds = project_dict.get('seeds', {})
  dbt_raw_version = project_dict.get('require-dbt-version', '>=0.0.0')
 
@@ -282,7 +278,6 @@ def from_project_config(cls, project_dict, packages_dict=None):
  models=models,
  on_run_start=on_run_start,
  on_run_end=on_run_end,
- archive=archive,
  seeds=seeds,
  dbt_version=dbt_version,
  packages=packages
@@ -329,7 +324,6 @@ def to_project_config(self, with_packages=False):
  'models': self.models,
  'on-run-start': self.on_run_start,
  'on-run-end': self.on_run_end,
- 'archive': self.archive,
  'seeds': self.seeds,
  'require-dbt-version': [
  v.to_version_string() for v in self.dbt_version

core/dbt/config/runtime.py

@@ -12,15 +12,6 @@
 from .project import Project
 
 
-_ARCHIVE_REMOVED_MESSAGE = '''
-The `archive` section in `dbt_project.yml` is no longer supported. Please use a
-`snapshot` block instead. For more information on snapshot blocks and a script
-to help migrate these archives, please consult the 0.14.0 migration guide:
-
-https://docs.getdbt.com/v0.14/docs/upgrading-to-014
-'''.strip()
-
-
 class RuntimeConfig(Project, Profile):
  """The runtime configuration, as constructed from its components. There's a
  lot because there is a lot of stuff!
@@ -29,8 +20,8 @@ def __init__(self, project_name, version, project_root, source_paths,
  macro_paths, data_paths, test_paths, analysis_paths,
  docs_paths, target_path, snapshot_paths, clean_targets,
  log_path, modules_path, quoting, models, on_run_start,
- on_run_end, archive, seeds, dbt_version, profile_name,
- target_name, config, threads, credentials, packages, args):
+ on_run_end, seeds, dbt_version, profile_name, target_name,
+ config, threads, credentials, packages, args):
  # 'vars'
  self.args = args
  self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
@@ -56,7 +47,6 @@ def __init__(self, project_name, version, project_root, source_paths,
  models=models,
  on_run_start=on_run_start,
  on_run_end=on_run_end,
- archive=archive,
  seeds=seeds,
  dbt_version=dbt_version,
  packages=packages
@@ -73,24 +63,19 @@ def __init__(self, project_name, version, project_root, source_paths,
  self.validate()
 
  @classmethod
- def from_parts(cls, project, profile, args, allow_archive_configs=False):
+ def from_parts(cls, project, profile, args):
  """Instantiate a RuntimeConfig from its components.
 
  :param profile Profile: A parsed dbt Profile.
  :param project Project: A parsed dbt Project.
  :param args argparse.Namespace: The parsed command-line arguments.
- :param allow_archive_configs bool: If True, ignore archive blocks in
- configs. This flag exists to enable archive migration.
  :returns RuntimeConfig: The new configuration.
  """
  quoting = deepcopy(
  get_relation_class_by_name(profile.credentials.type)
  .DEFAULTS['quote_policy']
  )
  quoting.update(project.quoting)
- if project.archive and not allow_archive_configs:
- # if the user has an `archive` section, raise an error
- raise DbtProjectError(_ARCHIVE_REMOVED_MESSAGE)
 
  return cls(
  project_name=project.project_name,
@@ -111,7 +96,6 @@ def from_parts(cls, project, profile, args, allow_archive_configs=False):
  models=project.models,
  on_run_start=project.on_run_start,
  on_run_end=project.on_run_end,
- archive=project.archive,
  seeds=project.seeds,
  dbt_version=project.dbt_version,
  packages=project.packages,
@@ -178,14 +162,12 @@ def validate(self):
  self.validate_version()
 
  @classmethod
- def from_args(cls, args, allow_archive_configs=False):
+ def from_args(cls, args):
  """Given arguments, read in dbt_project.yml from the current directory,
  read in packages.yml if it exists, and use them to find the profile to
  load.
 
  :param args argparse.Namespace: The arguments as parsed from the cli.
- :param allow_archive_configs bool: If True, ignore archive blocks in
- configs. This flag exists to enable archive migration.
  :raises DbtProjectError: If the project is invalid or missing.
  :raises DbtProfileError: If the profile is invalid or missing.
  :raises ValidationException: If the cli variables are invalid.
@@ -203,5 +185,4 @@ def from_args(cls, args, allow_archive_configs=False):
  project=project,
  profile=profile,
  args=args,
- allow_archive_configs=allow_archive_configs
  )

core/dbt/contracts/project.py

@@ -3,36 +3,6 @@
 from dbt.utils import deep_merge
 
 
-ARCHIVE_TABLE_CONFIG_CONTRACT = {
- 'type': 'object',
- 'additionalProperties': False,
- 'properties': {
- 'source_table': {'type': 'string'},
- 'target_table': {'type': 'string'},
- 'updated_at': {'type': 'string'},
- 'unique_key': {'type': 'string'},
- },
- 'required': ['source_table', 'target_table', 'updated_at', 'unique_key'],
-}
-
-
-ARCHIVE_CONFIG_CONTRACT = {
- 'type': 'object',
- 'additionalProperties': False,
- 'properties': {
- 'source_database': {'type': 'string'},
- 'target_database': {'type': 'string'},
- 'source_schema': {'type': 'string'},
- 'target_schema': {'type': 'string'},
- 'tables': {
- 'type': 'array',
- 'items': ARCHIVE_TABLE_CONFIG_CONTRACT,
- }
- },
- 'required': ['source_schema', 'target_schema', 'tables'],
-}
-
-
 PROJECT_CONTRACT = {
  'type': 'object',
  'description': 'The project configuration.',
@@ -138,10 +108,6 @@
  'type': 'array',
  'items': {'type': 'string'},
  },
- 'archive': {
- 'type': 'array',
- 'items': ARCHIVE_CONFIG_CONTRACT,
- },
  'seeds': {
  'type': 'object',
  'additionalProperties': True,

core/dbt/deprecations.py

@@ -40,17 +40,6 @@ class GenerateSchemaNameSingleArgDeprecated(DBTDeprecation):
  ''' # noqa
 
 
-class ArchiveDeprecated(DBTDeprecation):
- name = 'archives'
- description = '''As of dbt v0.14.0, the `dbt archive` command is renamed to
- `dbt snapshot` and "archives" are "snapshots". The `dbt archive` command will
- be removed in a future release.
-
- For more information, see:
- https://docs.getdbt.com/v0.14/docs/upgrading-to-014
- '''
-
-
 _adapter_renamed_description = """\
 The adapter function `adapter.{old_name}` is deprecated and will be removed in
  a future release of dbt. Please use `adapter.{new_name}` instead.
@@ -87,7 +76,6 @@ def warn(name, *args, **kwargs):
 deprecations_list = [
  DBTRepositoriesDeprecation(),
  GenerateSchemaNameSingleArgDeprecated(),
- ArchiveDeprecated(),
 ]
 
 deprecations = {d.name: d for d in deprecations_list}

core/dbt/main.py

@@ -23,7 +23,6 @@
 import dbt.task.freshness as freshness_task
 import dbt.task.run_operation as run_operation_task
 from dbt.task.list import ListTask
-from dbt.task.migrate import MigrationTask
 from dbt.task.rpc_server import RPCServerTask
 from dbt.adapters.factory import reset_adapters
 
@@ -328,19 +327,11 @@ def _build_deps_subparser(subparsers, base_subparser):
  return sub
 
 
-def _build_snapshot_subparser(subparsers, base_subparser, which='snapshot'):
- if which == 'archive':
- helpmsg = (
- 'DEPRECATED: This command is deprecated and will\n'
- 'be removed in a future release. Use dbt snapshot instead.'
- )
- else:
- helpmsg = 'Execute snapshots defined in your project'
-
+def _build_snapshot_subparser(subparsers, base_subparser):
  sub = subparsers.add_parser(
- which,
+ 'snapshot',
  parents=[base_subparser],
- help=helpmsg)
+ help='Execute snapshots defined in your project')
  sub.add_argument(
  '--threads',
  type=int,
@@ -350,7 +341,7 @@ def _build_snapshot_subparser(subparsers, base_subparser, which='snapshot'):
  settings in profiles.yml.
  """
  )
- sub.set_defaults(cls=snapshot_task.SnapshotTask, which=which)
+ sub.set_defaults(cls=snapshot_task.SnapshotTask, which='snapshot')
  return sub
 
 
@@ -633,38 +624,6 @@ def _build_run_operation_subparser(subparsers, base_subparser):
  return sub
 
 
-def _build_snapshot_migrate_subparser(subparsers, base_subparser):
- sub = subparsers.add_parser(
- 'snapshot-migrate',
- parents=[base_subparser],
- help='Run the snapshot migration script'
- )
- sub.add_argument(
- '--from-archive',
- action='store_true',
- help=('This flag is required for the 0.14.0 archive to snapshot '
- 'migration')
- )
- sub.add_argument(
- '--apply-files',
- action='store_true',
- dest='write_files',
- help='If set, write .sql files to disk instead of logging them'
- )
- sub.add_argument(
- '--apply-database',
- action='store_true',
- dest='migrate_database',
- help='If set, perform just the database migration'
- )
- sub.add_argument(
- '--apply',
- action='store_true',
- help='If set, implies --apply-database --apply-files'
- )
- sub.set_defaults(cls=MigrationTask, which='migration')
-
-
 def parse_args(args):
  p = DBTArgumentParser(
  prog='dbt',
@@ -750,10 +709,8 @@ def parse_args(args):
  _build_debug_subparser(subs, base_subparser)
  _build_deps_subparser(subs, base_subparser)
  _build_list_subparser(subs, base_subparser)
- _build_snapshot_migrate_subparser(subs, base_subparser)
 
  snapshot_sub = _build_snapshot_subparser(subs, base_subparser)
- archive_sub = _build_snapshot_subparser(subs, base_subparser, 'archive')
  rpc_sub = _build_rpc_subparser(subs, base_subparser)
  run_sub = _build_run_subparser(subs, base_subparser)
  compile_sub = _build_compile_subparser(subs, base_subparser)
@@ -763,8 +720,7 @@ def parse_args(args):
  _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub,
  rpc_sub)
  # --models, --exclude
- _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub,
- archive_sub)
+ _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub)
  _add_selection_arguments(snapshot_sub, models_name='select')
  # --full-refresh
  _add_table_mutability_arguments(run_sub, compile_sub)