add a simple set of graph unit tests for models only

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.5
+FROM python:3.6
 
 RUN apt-get update
 

Makefile

@@ -2,6 +2,10 @@
 
 changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ print $$2 }' | grep '\/test_[a-zA-Z_\-\.]\+.py'`
 
+it:
+ @echo "Unit test run starting..."
+ @time docker-compose run test tox -e unit-py27,pep8
+
 test:
  @echo "Full test run starting..."
  @time docker-compose run test tox

dbt/clients/system.py

@@ -0,0 +1,53 @@
+import fnmatch
+import os
+import os.path
+
+
+def find_matching(root_path,
+ relative_paths_to_search,
+ file_pattern):
+ """
+ Given an absolute `root_path`, a list of relative paths to that
+ absolute root path (`relative_paths_to_search`), and a `file_pattern`
+ like '*.sql', returns information about the files. For example:
+
+ > find_matching('/root/path', 'models', '*.sql')
+
+ [ { 'absolute_path': '/root/path/models/model_one.sql',
+ 'relative_path': 'models/model_one.sql',
+ 'searched_path': 'models' },
+ { 'absolute_path': '/root/path/models/subdirectory/model_two.sql',
+ 'relative_path': 'models/subdirectory/model_two.sql',
+ 'searched_path': 'models' } ]
+ """
+ matching = []
+
+ for relative_path_to_search in relative_paths_to_search:
+ absolute_path_to_search = os.path.join(
+ root_path, relative_path_to_search)
+ walk_results = os.walk(absolute_path_to_search)
+
+ for current_path, subdirectories, local_files in walk_results:
+ for local_file in local_files:
+ absolute_path = os.path.join(current_path, local_file)
+ relative_path = os.path.relpath(
+ absolute_path, absolute_path_to_search)
+
+ if fnmatch.fnmatch(local_file, file_pattern):
+ matching.append({
+ 'searched_path': relative_path_to_search,
+ 'absolute_path': absolute_path,
+ 'relative_path': relative_path,
+ })
+
+ return matching
+
+
+def load_file_contents(path, strip=True):
+ with open(path, 'rb') as handle:
+ to_return = handle.read().decode('utf-8')
+
+ if strip:
+ to_return = to_return.strip()
+
+ return to_return

dbt/compilation.py

@@ -6,10 +6,11 @@
 import sqlparse
 
 import dbt.project
+import dbt.utils
+
 from dbt.source import Source
 from dbt.utils import find_model_by_fqn, find_model_by_name, \
- dependency_projects, split_path, This, Var, compiler_error, \
- to_string
+ split_path, This, Var, compiler_error, to_string
 
 from dbt.linker import Linker
 from dbt.runtime import RuntimeContext
@@ -229,7 +230,7 @@ def wrapped_do_ref(*args):
 
  return wrapped_do_ref
 
- def get_context(self, linker, model,  models, add_dependency=False):
+ def get_context(self, linker, model, models, add_dependency=False):
  runtime = RuntimeContext(model=model)
 
  context = self.project.context()
@@ -272,10 +273,10 @@ def compile_model(self, linker, model, models, add_dependency=True):
  fs_loader = jinja2.FileSystemLoader(searchpath=model.root_dir)
  jinja = jinja2.Environment(loader=fs_loader)
 
- # this is a dumb jinja2 bug -- on windows, forward slashes
- # are EXPECTED
- posix_filepath = '/'.join(split_path(model.rel_filepath))
- template = jinja.get_template(posix_filepath)
+ template_contents = dbt.clients.system.load_file_contents(
+  model.absolute_path)
+
+ template = jinja.from_string(template_contents)
  context = self.get_context(
  linker, model, models, add_dependency=add_dependency
  )
@@ -521,7 +522,7 @@ def compile_archives(self):
 
  def get_models(self):
  all_models = self.model_sources(this_project=self.project)
- for project in dependency_projects(self.project):
+ for project in dbt.utils.dependency_projects(self.project):
  all_models.extend(
  self.model_sources(
  this_project=self.project, own_project=project
@@ -536,7 +537,7 @@ def compile(self, limit_to=None):
  all_models = self.get_models()
  all_macros = self.get_macros(this_project=self.project)
 
- for project in dependency_projects(self.project):
+ for project in dbt.utils.dependency_projects(self.project):
  all_macros.extend(
  self.get_macros(this_project=self.project, own_project=project)
  )

dbt/model.py

@@ -1,4 +1,3 @@
-
 import os.path
 import yaml
 import jinja2
@@ -202,6 +201,10 @@ def __init__(self, project, top_dir, rel_filepath, own_project):
 
  self.source_config = SourceConfig(project, own_project, self.fqn)
 
+ @property
+ def absolute_path(self):
+ return os.path.join(self.root_dir, self.rel_filepath)
+
  @property
  def root_dir(self):
  return os.path.join(self.own_project['project-root'], self.top_dir)
@@ -230,9 +233,7 @@ def serialize(self):
 
  @property
  def contents(self):
- filepath = os.path.join(self.root_dir, self.rel_filepath)
- with open(filepath) as fh:
- return fh.read().strip()
+ return dbt.clients.system.load_file_contents(self.absolute_path)
 
  @property
  def config(self):

dbt/source.py

@@ -3,6 +3,8 @@
 from dbt.model import Model, Analysis, TestModel, SchemaFile, Csv, Macro, \
  ArchiveModel, DataTest
 
+import dbt.clients.system
+
 
 class Source(object):
  def __init__(self, project, own_project=None):
@@ -15,72 +17,91 @@ def __init__(self, project, own_project=None):
  self.own_project_root = self.own_project['project-root']
  self.own_project_name = self.own_project['name']
 
- def find(self, source_paths, file_pattern):
- """returns abspath, relpath, filename of files matching file_regex in
- source_paths"""
- found = []
-
- if type(source_paths) not in (list, tuple):
- source_paths = [source_paths]
-
- for source_path in source_paths:
- root_path = os.path.join(self.own_project_root, source_path)
- for root, dirs, files in os.walk(root_path):
- for filename in files:
- abs_path = os.path.join(root, filename)
- rel_path = os.path.relpath(abs_path, root_path)
-
- if fnmatch.fnmatch(filename, file_pattern):
- found.append(
- (self.project,
- source_path,
- rel_path,
- self.own_project)
- )
- return found
+ def build_models_from_file_matches(
+ self,
+ to_build,
+ file_matches,
+ extra_args=[]):
+
+ build_args = [[self.project,
+ file_match.get('searched_path'),
+ file_match.get('relative_path'),
+ self.own_project] + extra_args
+ for file_match in file_matches]
+
+ return [to_build(*args) for args in build_args]
 
  def get_models(self, model_dirs, create_template):
- pattern = "[!.#~]*.sql"
- models = [Model(*model + (create_template,))
- for model in self.find(model_dirs, pattern)]
- return models
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ model_dirs,
+ "[!.#~]*.sql")
+
+ return self.build_models_from_file_matches(
+ Model,
+ file_matches,
+ [create_template])
 
  def get_test_models(self, model_dirs, create_template):
- pattern = "[!.#~]*.sql"
- models = [TestModel(*model + (create_template,))
- for model in self.find(model_dirs, pattern)]
- return models
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ model_dirs,
+ "[!.#~]*.sql")
+
+ return self.build_models_from_file_matches(
+ TestModel,
+ file_matches,
+ [create_template])
 
  def get_analyses(self, analysis_dirs):
- pattern = "[!.#~]*.sql"
- models = [Analysis(*analysis)
- for analysis in self.find(analysis_dirs, pattern)]
- return models
-
- def get_schemas(self, model_dirs):
- "Get schema.yml files"
- pattern = "[!.#~]*.yml"
- schemas = [SchemaFile(*schema)
- for schema in self.find(model_dirs, pattern)]
- return schemas
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ analysis_dirs,
+ "[!.#~]*.sql")
+
+ return self.build_models_from_file_matches(
+ Analysis,
+ file_matches)
+
+ def get_schemas(self, schema_dirs):
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ schema_dirs,
+ "[!.#~]*.yml")
+
+ return self.build_models_from_file_matches(
+ SchemaFile,
+ file_matches)
 
  def get_tests(self, test_dirs):
- "Get custom test files"
- pattern = "[!.#~]*.sql"
- tests = [DataTest(*test) for test in self.find(test_dirs, pattern)]
- return tests
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ test_dirs,
+ "[!.#~]*.sql")
+
+ return self.build_models_from_file_matches(
+ DataTest,
+ file_matches)
 
  def get_csvs(self, csv_dirs):
- "Get CSV files"
- pattern = "[!.#~]*.csv"
- csvs = [Csv(*csv) for csv in self.find(csv_dirs, pattern)]
- return csvs
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ csv_dirs,
+ "[!.#~]*.csv")
+
+ return self.build_models_from_file_matches(
+ Csv,
+ file_matches)
 
  def get_macros(self, macro_dirs):
- "Get Macro files"
- pattern = "[!.#~]*.sql"
- macros = [Macro(*macro) for macro in self.find(macro_dirs, pattern)]
- return macros
+ file_matches = dbt.clients.system.find_matching(
+ self.own_project_root,
+ macro_dirs,
+ "[!.#~]*.sql")
+
+ return self.build_models_from_file_matches(
+ Macro,
+ file_matches)
 
  def get_archives(self, create_template):
  "Get Archive models defined in project config"

dev_requirements.txt

@@ -1,5 +1,4 @@
 nose>=1.3.7
-nosy>=1.1.2
 mock>=1.3.0
 pep8>=1.6.2
 bumpversion==0.5.3

test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py

@@ -91,8 +91,7 @@ def project_config(self):
  "vars": {
  "config_1": "ghi",
  "config_2": "jkl",
- "bool_config": True
-
+ "bool_config": True,
  }
  }
  }

test/integration/009_data_tests_test/test_data_tests.py

@@ -43,7 +43,6 @@ def test_data_tests(self):
  self.run_dbt()
  test_results = self.run_data_validations()
 
-
  for result in test_results:
  # assert that all deliberately failing tests actually fail
  if 'fail' in result.model.name: