Feature/archive blocks

.circleci/config.yml

@@ -19,7 +19,7 @@ jobs:
  PGUSER: root
  PGPASSWORD: password
  PGDATABASE: postgres
- - run: tox -e pep8,unit-py27,unit-py36
+ - run: tox -e flake8,unit-py27,unit-py36
  integration-postgres-py36:
  docker: *test_and_postgres
  steps:

CONTRIBUTING.md

@@ -76,8 +76,8 @@ A short list of tools used in dbt testing that will be helpful to your understan
 - [tox](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions
 - [nosetests](http://nose.readthedocs.io/en/latest) to discover/run tests
 - [make](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
-- [pep8](https://pep8.readthedocs.io/en/release-1.7.x/) for code linting
-- [CircleCI](https://circleci.com/product/) and [Appveyor](https://www.appveyor.com/docs/)
+- [flake8](https://gitlab.com/pycqa/flake8) for code linting
+- [CircleCI](https://circleci.com/product/) and [Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/)
 
 If you're unfamiliar with any or all of these, that's fine! You really do not have to have a deep understanding of any of these to get by.
 

Makefile

@@ -11,7 +11,7 @@ test:
 
 test-unit:
  @echo "Unit test run starting..."
- @time docker-compose run test tox -e unit-py27,unit-py36,pep8
+ @time docker-compose run test tox -e unit-py27,unit-py36,flake8
 
 test-integration:
  @echo "Integration test run starting..."

core/dbt/adapters/base/__init__.py

@@ -1,5 +1,8 @@
-from dbt.adapters.base.meta import available
-from dbt.adapters.base.relation import BaseRelation
-from dbt.adapters.base.connections import BaseConnectionManager, Credentials
-from dbt.adapters.base.impl import BaseAdapter
-from dbt.adapters.base.plugin import AdapterPlugin
+# these are all just exports, #noqa them so flake8 will be happy
+from dbt.adapters.base.meta import available # noqa
+from dbt.adapters.base.relation import BaseRelation # noqa
+from dbt.adapters.base.relation import Column # noqa
+from dbt.adapters.base.connections import BaseConnectionManager # noqa
+from dbt.adapters.base.connections import Credentials # noqa
+from dbt.adapters.base.impl import BaseAdapter # noqa
+from dbt.adapters.base.plugin import AdapterPlugin # noqa

core/dbt/adapters/base/impl.py

@@ -7,18 +7,17 @@
 
 import dbt.exceptions
 import dbt.flags
-import dbt.schema
 import dbt.clients.agate_helper
 
 from dbt.compat import abstractclassmethod, classmethod
 from dbt.node_types import NodeType
 from dbt.loader import GraphLoader
 from dbt.logger import GLOBAL_LOGGER as logger
-from dbt.schema import Column
 from dbt.utils import filter_null_values
 
 from dbt.adapters.base.meta import AdapterMeta, available, available_deprecated
 from dbt.adapters.base import BaseRelation
+from dbt.adapters.base import Column
 from dbt.adapters.cache import RelationsCache
 
 
@@ -534,6 +533,48 @@ def get_missing_columns(self, from_relation, to_relation):
  if col_name in missing_columns
  ]
 
+ @available
+ def valid_archive_target(self, relation):
+ """Ensure that the target relation is valid, by making sure it has the
+ expected columns.
+
+ :param Relation relation: The relation to check
+ :raises dbt.exceptions.CompilationException: If the columns are
+ incorrect.
+ """
+ if not isinstance(relation, self.Relation):
+ dbt.exceptions.invalid_type_error(
+ method_name='is_existing_old_style_archive',
+ arg_name='relation',
+ got_value=relation,
+ expected_type=self.Relation)
+
+ columns = self.get_columns_in_relation(relation)
+ names = set(c.name.lower() for c in columns)
+ expanded_keys = ('scd_id', 'valid_from', 'valid_to')
+ extra = []
+ missing = []
+ for legacy in expanded_keys:
+ desired = 'dbt_' + legacy
+ if desired not in names:
+ missing.append(desired)
+ if legacy in names:
+ extra.append(legacy)
+
+ if missing:
+ if extra:
+ msg = (
+ 'Archive target has ("{}") but not ("{}") - is it an '
+ 'unmigrated previous version archive?'
+ .format('", "'.join(extra), '", "'.join(missing))
+ )
+ else:
+ msg = (
+ 'Archive target is not an archive table (missing "{}")'
+ .format('", "'.join(missing))
+ )
+ dbt.exceptions.raise_compiler_error(msg)
+
  @available
  def expand_target_column_types(self, temp_table, to_relation):
  if not isinstance(to_relation, self.Relation):

core/dbt/adapters/base/plugin.py

@@ -1,5 +1,3 @@
-import os
-
 from dbt.config.project import Project
 
 

core/dbt/adapters/base/relation.py

@@ -303,3 +303,91 @@ def is_cte(self):
  @property
  def is_view(self):
  return self.type == self.View
+
+
+class Column(object):
+ TYPE_LABELS = {
+ 'STRING': 'TEXT',
+ 'TIMESTAMP': 'TIMESTAMP',
+ 'FLOAT': 'FLOAT',
+ 'INTEGER': 'INT'
+ }
+
+ def __init__(self, column, dtype, char_size=None, numeric_precision=None,
+ numeric_scale=None):
+ self.column = column
+ self.dtype = dtype
+ self.char_size = char_size
+ self.numeric_precision = numeric_precision
+ self.numeric_scale = numeric_scale
+
+ @classmethod
+ def translate_type(cls, dtype):
+ return cls.TYPE_LABELS.get(dtype.upper(), dtype)
+
+ @classmethod
+ def create(cls, name, label_or_dtype):
+ column_type = cls.translate_type(label_or_dtype)
+ return cls(name, column_type)
+
+ @property
+ def name(self):
+ return self.column
+
+ @property
+ def quoted(self):
+ return '"{}"'.format(self.column)
+
+ @property
+ def data_type(self):
+ if self.is_string():
+ return Column.string_type(self.string_size())
+ elif self.is_numeric():
+ return Column.numeric_type(self.dtype, self.numeric_precision,
+ self.numeric_scale)
+ else:
+ return self.dtype
+
+ def is_string(self):
+ return self.dtype.lower() in ['text', 'character varying', 'character',
+ 'varchar']
+
+ def is_numeric(self):
+ return self.dtype.lower() in ['numeric', 'number']
+
+ def string_size(self):
+ if not self.is_string():
+ raise RuntimeError("Called string_size() on non-string field!")
+
+ if self.dtype == 'text' or self.char_size is None:
+ # char_size should never be None. Handle it reasonably just in case
+ return 255
+ else:
+ return int(self.char_size)
+
+ def can_expand_to(self, other_column):
+ """returns True if this column can be expanded to the size of the
+ other column"""
+ if not self.is_string() or not other_column.is_string():
+ return False
+
+ return other_column.string_size() > self.string_size()
+
+ def literal(self, value):
+ return "{}::{}".format(value, self.data_type)
+
+ @classmethod
+ def string_type(cls, size):
+ return "character varying({})".format(size)
+
+ @classmethod
+ def numeric_type(cls, dtype, precision, scale):
+ # This could be decimal(...), numeric(...), number(...)
+ # Just use whatever was fed in here -- don't try to get too clever
+ if precision is None or scale is None:
+ return dtype
+ else:
+ return "{}({},{})".format(dtype, precision, scale)
+
+ def __repr__(self):
+ return "<Column {} ({})>".format(self.name, self.data_type)

core/dbt/adapters/sql/__init__.py

@@ -1,2 +1,3 @@
-from dbt.adapters.sql.connections import SQLConnectionManager
-from dbt.adapters.sql.impl import SQLAdapter
+# these are all just exports, #noqa them so flake8 will be happy
+from dbt.adapters.sql.connections import SQLConnectionManager # noqa
+from dbt.adapters.sql.impl import SQLAdapter # noqa

core/dbt/api/object.py

@@ -1,6 +1,6 @@
 import copy
 from collections import Mapping
-from jsonschema import Draft4Validator
+from jsonschema import Draft7Validator
 
 from dbt.exceptions import JSONValidationException
 from dbt.utils import deep_merge
@@ -79,7 +79,7 @@ def validate(self):
  of this instance. If any attributes are missing or
  invalid, raise a ValidationException.
  """
- validator = Draft4Validator(self.SCHEMA)
+ validator = Draft7Validator(self.SCHEMA)
 
  errors = set() # make errors a set to avoid duplicates
 

core/dbt/clients/_jinja_blocks.py

@@ -174,8 +174,8 @@ def _expect_match(self, expected_name, *patterns, **kwargs):
  match = self._first_match(*patterns, **kwargs)
  if match is None:
  msg = 'unexpected EOF, expected {}, got "{}"'.format(
-  expected_name, self.data[self.pos:]
-  )
+ expected_name, self.data[self.pos:]
+ )
  dbt.exceptions.raise_compiler_error(msg)
  return match
 

core/dbt/clients/jinja.py

@@ -11,6 +11,7 @@
 
 import dbt.compat
 import dbt.exceptions
+import dbt.utils
 
 from dbt.clients._jinja_blocks import BlockIterator
 
@@ -55,7 +56,7 @@ def _compile(self, source, filename):
  linecache.cache[filename] = (
  len(source),
  None,
- [line+'\n' for line in source.splitlines()],
+ [line + '\n' for line in source.splitlines()],
  filename
  )
 

core/dbt/compilation.py

@@ -1,14 +1,11 @@
 import itertools
 import os
-import json
-from collections import OrderedDict, defaultdict
-import sqlparse
+from collections import defaultdict
 
 import dbt.utils
 import dbt.include
 import dbt.tracking
 
-from dbt import deprecations
 from dbt.utils import get_materialization, NodeType, is_type
 from dbt.linker import Linker
 
@@ -19,9 +16,8 @@
 import dbt.flags
 import dbt.loader
 import dbt.config
-from dbt.contracts.graph.compiled import CompiledNode, CompiledGraph
+from dbt.contracts.graph.compiled import CompiledNode
 
-from dbt.clients.system import write_json
 from dbt.logger import GLOBAL_LOGGER as logger
 
 graph_file_name = 'graph.gpickle'

core/dbt/config/__init__.py

@@ -1,5 +1,5 @@
-
-from .renderer import ConfigRenderer
-from .profile import Profile, UserConfig, PROFILES_DIR
-from .project import Project
-from .runtime import RuntimeConfig
+# all these are just exports, they need "noqa" so flake8 will not complain.
+from .renderer import ConfigRenderer # noqa
+from .profile import Profile, UserConfig, PROFILES_DIR # noqa
+from .project import Project # noqa
+from .runtime import RuntimeConfig # noqa

core/dbt/config/project.py

@@ -1,4 +1,3 @@
-
 from copy import deepcopy
 import hashlib
 import os
@@ -14,7 +13,6 @@
 from dbt.exceptions import SemverException
 from dbt.exceptions import ValidationException
 from dbt.exceptions import warn_or_error
-from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.semver import VersionSpecifier
 from dbt.semver import versions_compatible
 from dbt.version import get_installed_version
@@ -145,9 +143,10 @@ def _parse_versions(versions):
 class Project(object):
  def __init__(self, project_name, version, project_root, profile_name,
  source_paths, macro_paths, data_paths, test_paths,
- analysis_paths, docs_paths, target_path, clean_targets,
- log_path, modules_path, quoting, models, on_run_start,
- on_run_end, archive, seeds, dbt_version, packages):
+ analysis_paths, docs_paths, target_path, archive_paths,
+ clean_targets, log_path, modules_path, quoting, models,
+ on_run_start, on_run_end, archive, seeds, dbt_version,
+ packages):
  self.project_name = project_name
  self.version = version
  self.project_root = project_root
@@ -159,6 +158,7 @@ def __init__(self, project_name, version, project_root, profile_name,
  self.analysis_paths = analysis_paths
  self.docs_paths = docs_paths
  self.target_path = target_path
+ self.archive_paths = archive_paths
  self.clean_targets = clean_targets
  self.log_path = log_path
  self.modules_path = modules_path
@@ -241,6 +241,7 @@ def from_project_config(cls, project_dict, packages_dict=None):
  analysis_paths = project_dict.get('analysis-paths', [])
  docs_paths = project_dict.get('docs-paths', source_paths[:])
  target_path = project_dict.get('target-path', 'target')
+ archive_paths = project_dict.get('archive-paths', ['archives'])
  # should this also include the modules path by default?
  clean_targets = project_dict.get('clean-targets', [target_path])
  log_path = project_dict.get('log-path', 'logs')
@@ -274,6 +275,7 @@ def from_project_config(cls, project_dict, packages_dict=None):
  analysis_paths=analysis_paths,
  docs_paths=docs_paths,
  target_path=target_path,
+ archive_paths=archive_paths,
  clean_targets=clean_targets,
  log_path=log_path,
  modules_path=modules_path,
@@ -321,6 +323,7 @@ def to_project_config(self, with_packages=False):
  'analysis-paths': self.analysis_paths,
  'docs-paths': self.docs_paths,
  'target-path': self.target_path,
+ 'archive-paths': self.archive_paths,
  'clean-targets': self.clean_targets,
  'log-path': self.log_path,
  'quoting': self.quoting,

core/dbt/config/runtime.py

@@ -18,10 +18,10 @@ class RuntimeConfig(Project, Profile):
  """
  def __init__(self, project_name, version, project_root, source_paths,
  macro_paths, data_paths, test_paths, analysis_paths,
- docs_paths, target_path, clean_targets, log_path,
- modules_path, quoting, models, on_run_start, on_run_end,
- archive, seeds, dbt_version, profile_name, target_name,
- config, threads, credentials, packages, args):
+ docs_paths, target_path, archive_paths, clean_targets,
+ log_path, modules_path, quoting, models, on_run_start,
+ on_run_end, archive, seeds, dbt_version, profile_name,
+ target_name, config, threads, credentials, packages, args):
  # 'vars'
  self.args = args
  self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
@@ -39,6 +39,7 @@ def __init__(self, project_name, version, project_root, source_paths,
  analysis_paths=analysis_paths,
  docs_paths=docs_paths,
  target_path=target_path,
+ archive_paths=archive_paths,
  clean_targets=clean_targets,
  log_path=log_path,
  modules_path=modules_path,
@@ -87,6 +88,7 @@ def from_parts(cls, project, profile, args):
  analysis_paths=project.analysis_paths,
  docs_paths=project.docs_paths,
  target_path=project.target_path,
+ archive_paths=project.archive_paths,
  clean_targets=project.clean_targets,
  log_path=project.log_path,
  modules_path=project.modules_path,