Merge remote-tracking branch 'origin/0.12.latest' into dev/stephen-gi…

…rard

CHANGELOG.md

@@ -7,6 +7,39 @@ This release makes dbt and its adapters into a core-and-plugin architecture.
 ### Breaking Changes
 - '{{this}}' is no longer respected in hooks [#1176](https:/fishtown-analytics/dbt/pull/1176), implementing [#878](https:/fishtown-analytics/dbt/issues/878)
 
+## dbt 0.12.2 - Grace Kelly (January 8, 2019)
+
+### Overview
+
+This release reduces the runtime of dbt projects by improving dbt's approach to model running. Additionally, a number of workflow improvements have been added. 
+
+### Deprecations
+- Deprecate `sql_where` ([#744](https:/fishtown-analytics/dbt/issues/744)) ([docs](https://docs.getdbt.com/v0.12/docs/configuring-incremental-models))
+
+### Features
+- More intelligently order and execute nodes in the graph. This _significantly_ speeds up the runtime of most dbt projects ([#813](https:/fishtown-analytics/dbt/issues/813))
+- Add `-m` flag as an alias for `--models` ([#1160](https:/fishtown-analytics/dbt/issues/1160))
+- Add `post_hook` and `pre_hook` as aliases for `post-hook` and `pre-hook`, respectively ([#1124](https:/fishtown-analytics/dbt/issues/1124)) ([docs](https://docs.getdbt.com/v0.12/docs/using-hooks))
+- Better handling of git errors in `dbt deps` + full support for Windows ([#994](https:/fishtown-analytics/dbt/issues/994), [#778](https:/fishtown-analytics/dbt/issues/778), [#895](https:/fishtown-analytics/dbt/issues/895))
+- Add support for specifying a `location` in BigQuery datasets ([#969](https:/fishtown-analytics/dbt/issues/969)) ([docs](https://docs.getdbt.com/v0.12/docs/supported-databases#section-dataset-locations))
+- Add support for Jinja expressions using the `{% do ... %}` block ([#1113](https:/fishtown-analytics/dbt/issues/1113))
+- The `dbt debug` command is actually useful now ([#1061](https:/fishtown-analytics/dbt/issues/1061))
+- The `config` function can now be called multiple times in a model ([#558](https:/fishtown-analytics/dbt/issues/558))
+- Source the latest version of dbt from PyPi instead of GitHub ([#1122](https:/fishtown-analytics/dbt/issues/1122))
+- Add a peformance profiling mechnanism to dbt ([#1001](https:/fishtown-analytics/dbt/issues/1001))
+- Add caching for dbt's macros-only manifest to speedup parsing ([#1098](https:/fishtown-analytics/dbt/issues/1098))
+
+### Fixes
+- Fix for custom schemas used alongside the `generate_schema_name` macro ([#801](https:/fishtown-analytics/dbt/issues/801))
+- Fix for silent failure of tests that reference nonexistent models ([#968](https:/fishtown-analytics/dbt/issues/968))
+- Fix for `generate_schema_name` macros that return whitespace-padded schema names ([#1074](https:/fishtown-analytics/dbt/issues/1074))
+- Fix for incorrect relation type for backup tables on Snowflake ([#1103](https:/fishtown-analytics/dbt/issues/1103))
+- Fix for incorrectly cased values in the relation cache ([#1140](https:/fishtown-analytics/dbt/issues/1140))
+- Fix for JSON decoding error on Python2 installed with Anaconda ([#1155](https:/fishtown-analytics/dbt/issues/1155))
+- Fix for unhandled exceptions that occur in anonymous event tracking ([#1180](https:/fishtown-analytics/dbt/issues/1180))
+- Fix for analysis files that contain `raw` tags ([#1152](https:/fishtown-analytics/dbt/issues/1152))
+- Fix for packages which reference the [hubsite](hub.getdbt.com) ([#1095](https:/fishtown-analytics/dbt/issues/1095))
+
 ## dbt 0.12.1 - (November 15, 2018)
 
 ### Overview

core/dbt/runner.py

@@ -54,6 +54,7 @@ def __init__(self, config, query, Runner):
  ])
  self.node_results = []
  self._skipped_children = {}
+ self._raise_next_tick = None
 
  def get_runner(self, node):
  adapter = get_adapter(self.config)
@@ -82,7 +83,11 @@ def call_runner(self, runner):
  runner.after_execute(result)
 
  if result.errored and runner.raise_on_first_error():
- raise dbt.exceptions.RuntimeException(result.error)
+ # if we raise inside a thread, it'll just get silently swallowed.
+ # stash the error message we want on the RunManager, and it will
+ # check the next 'tick' - should be soon since our thread is about
+ # to finish!
+ self._raise_next_tick = result.error
 
  return result
 
@@ -100,16 +105,23 @@ def _submit(self, pool, args, callback):
  else:
  pool.apply_async(self.call_runner, args=args, callback=callback)
 
+ def _raise_set_error(self):
+ if self._raise_next_tick is not None:
+ raise dbt.exceptions.RuntimeException(self._raise_next_tick)
+
  def run_queue(self, pool):
  """Given a pool, submit jobs from the queue to the pool.
  """
  def callback(result):
- """A callback to handle results."""
+ """Note: mark_done, at a minimum, must happen here or dbt will
+ deadlock during ephemeral result error handling!
+ """
  self._handle_result(result)
  self.job_queue.mark_done(result.node.unique_id)
 
  while not self.job_queue.empty():
  node = self.job_queue.get()
+ self._raise_set_error()
  runner = self.get_runner(node)
  # we finally know what we're running! Make sure we haven't decided
  # to skip it due to upstream failures
@@ -118,14 +130,18 @@ def callback(result):
  runner.do_skip(cause=cause)
  args = (runner,)
  self._submit(pool, args, callback)
+
  # block on completion
  self.job_queue.join()
+ # if an error got set during join(), raise it.
+ self._raise_set_error()
 
  return
 
  def _handle_result(self, result):
- """Note: this happens inside an apply_async() callback, so it must be
- "fast". (The pool worker thread will block!)
+ """Mark the result as completed, insert the `CompiledResultNode` into
+ the manifest, and mark any descendants (potentially with a 'cause' if
+ the result was an ephemeral model) as skipped.
  """
  is_ephemeral = self.Runner.is_ephemeral_model(result.node)
  if not is_ephemeral:

core/dbt/task/debug.py

@@ -117,35 +117,35 @@ def run(self):
  def _load_project(self):
  if not os.path.exists(self.project_path):
  self.project_fail_details = FILE_NOT_FOUND
- return red('✗ not found')
+ return red('ERROR not found')
 
  try:
  self.project = Project.from_current_directory(self.cli_vars)
  except dbt.exceptions.DbtConfigError as exc:
  self.project_fail_details = str(exc)
- return red('✗ invalid')
+ return red('ERROR invalid')
 
- return green('✓ found and valid')
+ return green('OK found and valid')
 
  def _profile_found(self):
  if not self.raw_profile_data:
- return red('✗ not found')
+ return red('ERROR not found')
  if self.profile_name in self.raw_profile_data:
- return green('✓ found')
+ return green('OK found')
  else:
- return red('✗ not found')
+ return red('ERROR not found')
 
  def _target_found(self):
  requirements = (self.raw_profile_data and self.profile_name and
  self.target_name)
  if not requirements:
- return red('✗ not found')
+ return red('ERROR not found')
  if self.profile_name not in self.raw_profile_data:
- return red('✗ not found')
+ return red('ERROR not found')
  profiles = self.raw_profile_data[self.profile_name]['outputs']
  if self.target_name not in profiles:
- return red('✗ not found')
- return green('✓ found')
+ return red('ERROR not found')
+ return green('OK found')
 
  def _choose_profile_name(self):
  assert self.project or self.project_fail_details, \
@@ -195,7 +195,7 @@ def _load_profile(self):
  self.messages.append(MISSING_PROFILE_MESSAGE.format(
  path=self.profile_path, url=ProfileConfigDocs
  ))
- return red('✗ not found')
+ return red('ERROR not found')
 
  try:
  raw_profile_data = load_yaml_text(
@@ -214,17 +214,17 @@ def _load_profile(self):
  self.cli_vars)
  except dbt.exceptions.DbtConfigError as exc:
  self.profile_fail_details = str(exc)
- return red('✗ invalid')
+ return red('ERROR invalid')
 
- return green('✓ found and valid')
+ return green('OK found and valid')
 
  def test_git(self):
  try:
  dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help'])
  except dbt.exceptions.ExecutableError as exc:
  self.messages.append('Error from git --help: {!s}'.format(exc))
- return red('✗ error')
- return green('✓ found')
+ return red('ERROR')
+ return green('OK found')
 
  def test_dependencies(self):
  print('Required dependencies:')
@@ -276,8 +276,8 @@ def _connection_result(self):
  err=str(exc),
  url=ProfileConfigDocs
  ))
- return red('✗ error')
- return green('✓ connection ok')
+ return red('ERROR')
+ return green('OK connection ok')
 
  def test_connection(self):
  if not self.profile:

core/dbt/task/init.py

@@ -64,7 +64,6 @@ def clone_starter_repo(self, project_name):
  dbt.clients.git.clone(
  STARTER_REPO, '.', project_name,
  remove_git_dir=True)
- dbt.clients.git.remove_remote(project_name)
 
  def create_profiles_dir(self, profiles_dir):
  if not os.path.exists(profiles_dir):

test/integration/040_init_test/test_init.py

@@ -0,0 +1,41 @@
+
+from test.integration.base import DBTIntegrationTest, use_profile
+import os
+import shutil
+
+
+class TestInit(DBTIntegrationTest):
+ def tearDown(self):
+ project_name = self.get_project_name()
+
+ if os.path.exists(project_name):
+ shutil.rmtree(project_name)
+
+ super(TestInit, self).tearDown()
+
+ def get_project_name(self):
+ return "my_project_{}".format(self.unique_schema())
+
+ @property
+ def schema(self):
+ return "init_040"
+
+ @property
+ def models(self):
+ return "test/integration/040_init_test/models"
+
+ @use_profile('postgres')
+ def test_postgres_init_task(self):
+ project_name = self.get_project_name()
+ self.run_dbt(['init', project_name])
+
+ dir_exists = os.path.exists(project_name)
+ project_file = os.path.join(project_name, 'dbt_project.yml')
+ project_file_exists = os.path.exists(project_file)
+
+ git_dir = os.path.join(project_name, '.git')
+ git_dir_exists = os.path.exists(git_dir)
+
+ self.assertTrue(dir_exists)
+ self.assertTrue(project_file_exists)
+ self.assertFalse(git_dir_exists)