Better error messages (#445)

* wip

* overhauled yaml validation error messages

* use yaml client (mostly) everywhere

* fix imports

* fix yaml client namespace for python2

* pep8

* code cleanup + typos

dbt/clients/yaml_helper.py

@@ -0,0 +1,57 @@
+import dbt.compat
+import dbt.exceptions
+
+import yaml
+import yaml.scanner
+
+
+YAML_ERROR_MESSAGE = """
+Syntax error near line {line_number}
+------------------------------
+{nice_error}
+
+Raw Error:
+------------------------------
+{raw_error}
+""".strip()
+
+
+def line_no(i, line, width=3):
+ line_number = dbt.compat.to_string(i).ljust(width)
+ return "{}| {}".format(line_number, line)
+
+
+def prefix_with_line_numbers(string, no_start, no_end):
+ line_list = string.split('\n')
+
+ numbers = range(no_start, no_end)
+ relevant_lines = line_list[no_start:no_end]
+
+ return "\n".join([
+ line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines)
+ ])
+
+
+def contextualized_yaml_error(raw_contents, error):
+ mark = error.problem_mark
+
+ min_line = max(mark.line - 3, 0)
+ max_line = mark.line + 4
+
+ nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line)
+
+ return YAML_ERROR_MESSAGE.format(line_number=mark.line + 1,
+ nice_error=nice_error,
+ raw_error=error)
+
+
+def load_yaml_text(contents):
+ try:
+ return yaml.safe_load(contents)
+ except (yaml.scanner.ScannerError, yaml.YAMLError) as e:
+ if hasattr(e, 'problem_mark'):
+ error = contextualized_yaml_error(contents, e)
+ else:
+ error = dbt.compat.to_string(e)
+
+ raise dbt.exceptions.ValidationException(error)

dbt/config.py

@@ -1,24 +1,30 @@
 import os.path
-import yaml
-import yaml.scanner
 
 import dbt.exceptions
+import dbt.clients.yaml_helper
+import dbt.clients.system
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
 
+INVALID_PROFILE_MESSAGE = """
+dbt encountered an error while trying to read your profiles.yml file.
+
+{error_string}
+"""
+
+
 def read_profile(profiles_dir):
- # TODO: validate profiles_dir
  path = os.path.join(profiles_dir, 'profiles.yml')
 
+ contents = None
  if os.path.isfile(path):
  try:
- with open(path, 'r') as f:
- return yaml.safe_load(f)
- except (yaml.scanner.ScannerError,
- yaml.YAMLError) as e:
- raise dbt.exceptions.ValidationException(
- ' Could not read {}\n\n{}'.format(path, str(e)))
+ contents = dbt.clients.system.load_file_contents(path, strip=False)
+ return dbt.clients.yaml_helper.load_yaml_text(contents)
+ except dbt.exceptions.ValidationException as e:
+ msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
+ raise dbt.exceptions.ValidationException(msg)
 
  return {}
 

dbt/main.py

@@ -20,6 +20,13 @@
 import dbt.config as config
 import dbt.adapters.cache as adapter_cache
 import dbt.ui.printer
+import dbt.compat
+
+PROFILES_HELP_MESSAGE = """
+For more information on configuring profiles, please consult the dbt docs:
+
+https://dbt.readme.io/docs/configure-your-profile
+"""
 
 
 def main(args=None):
@@ -143,17 +150,19 @@ def invoke_dbt(parsed):
  proj.validate()
  except project.DbtProjectError as e:
  logger.info("Encountered an error while reading the project:")
- logger.info(" ERROR {}".format(str(e)))
- logger.info(
- "Did you set the correct --profile? Using: {}"
- .format(parsed.profile))
-
- logger.info("Valid profiles:")
+ logger.info(dbt.compat.to_string(e))
 
  all_profiles = project.read_profiles(parsed.profiles_dir).keys()
 
- for profile in all_profiles:
- logger.info(" - {}".format(profile))
+ if len(all_profiles) > 0:
+ logger.info("Defined profiles:")
+ for profile in all_profiles:
+ logger.info(" - {}".format(profile))
+ else:
+ logger.info("There are no profiles defined in your "
+ "profiles.yml file")
+
+ logger.info(PROFILES_HELP_MESSAGE)
 
  dbt.tracking.track_invalid_invocation(
  project=proj,

dbt/parser.py

@@ -1,6 +1,5 @@
 import copy
 import os
-import yaml
 import re
 
 import dbt.flags
@@ -9,6 +8,7 @@
 
 import jinja2.runtime
 import dbt.clients.jinja
+import dbt.clients.yaml_helper
 
 import dbt.contracts.graph.parsed
 import dbt.contracts.graph.unparsed
@@ -426,7 +426,14 @@ def parse_schema_tests(tests, root_project, projects):
  to_return = {}
 
  for test in tests:
- test_yml = yaml.safe_load(test.get('raw_yml'))
+ raw_yml = test.get('raw_yml')
+ test_name = "{}:{}".format(test.get('package_name'), test.get('path'))
+
+ try:
+ test_yml = dbt.clients.yaml_helper.load_yaml_text(raw_yml)
+ except dbt.exceptions.ValidationException as e:
+ test_yml = None
+ logger.info("Error reading {} - Skipping\n{}".format(test_name, e))
 
  if test_yml is None:
  continue
@@ -551,7 +558,7 @@ def load_and_parse_yml(package_name, root_project, all_projects, root_dir,
 
  for file_match in file_matches:
  file_contents = dbt.clients.system.load_file_contents(
- file_match.get('absolute_path'))
+ file_match.get('absolute_path'), strip=False)
 
  parts = dbt.utils.split_path(file_match.get('relative_path', ''))
  name, _ = os.path.splitext(parts[-1])

dbt/project.py

@@ -1,14 +1,13 @@
 import os.path
-import yaml
 import pprint
 import copy
-import sys
 import hashlib
 import re
-from voluptuous import Schema, Required, Invalid
+from voluptuous import Required, Invalid
 
 import dbt.deprecations
 import dbt.contracts.connection
+import dbt.clients.yaml_helper
 from dbt.logger import GLOBAL_LOGGER as logger
 
 default_project_cfg = {
@@ -30,6 +29,19 @@
 
 default_profiles_dir = os.path.join(os.path.expanduser('~'), '.dbt')
 
+NO_SUPPLIED_PROFILE_ERROR = """\
+dbt cannot run because no profile was specified for this dbt project.
+To specify a profile for this project, add a line like the this to
+your dbt_project.yml file:
+
+profile: [profile name]
+
+Here, [profile name] should be replaced with a profile name
+defined in your profiles.yml file. You can find profiles.yml here:
+
+{profiles_file}/profiles.yml
+""".format(profiles_file=default_profiles_dir)
+
 
 class DbtProjectError(Exception):
  def __init__(self, message, project):
@@ -60,9 +72,7 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None,
  self.profile_to_load = self.cfg['profile']
 
  if self.profile_to_load is None:
- raise DbtProjectError(
- "No profile was supplied in the dbt_project.yml file, or the "
- "command line", self)
+ raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR, self)
 
  if self.profile_to_load in self.profiles:
  self.cfg.update(self.profiles[self.profile_to_load])
@@ -187,7 +197,7 @@ def read_project(filename, profiles_dir=None, validate=True,
 
  project_file_contents = dbt.clients.system.load_file_contents(filename)
 
- project_cfg = yaml.safe_load(project_file_contents)
+ project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents)
  project_cfg['project-root'] = os.path.dirname(
  os.path.abspath(filename))
  profiles = read_profiles(profiles_dir)