-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a new ListTask, and 'dbt list'/'dbt ls'
All tasks now have a 'pre_init_hook' classmethod, called by main - runs after args are parsed, before anything else
- Loading branch information
Jacob Beck
committed
May 2, 2019
1 parent
4c02b4a
commit e043643
Showing
14 changed files
with
524 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
from __future__ import print_function | ||
|
||
import json | ||
|
||
from dbt.task.runnable import GraphRunnableTask, ManifestTask | ||
from dbt.node_types import NodeType | ||
import dbt.exceptions | ||
from dbt.logger import GLOBAL_LOGGER as logger | ||
from dbt.logger import log_to_stderr | ||
|
||
|
||
class ListTask(GraphRunnableTask): | ||
DEFAULT_RESOURCE_VALUES = frozenset(( | ||
NodeType.Model, | ||
NodeType.Archive, | ||
NodeType.Seed, | ||
NodeType.Test, | ||
NodeType.Source, | ||
)) | ||
ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset(( | ||
NodeType.Analysis, | ||
)) | ||
ALLOWED_KEYS = frozenset(( | ||
'alias', | ||
'name', | ||
'package_name', | ||
'depends_on', | ||
'tags', | ||
'config', | ||
'resource_type', | ||
'source_name', | ||
)) | ||
|
||
def __init__(self, args, config): | ||
super(ListTask, self).__init__(args, config) | ||
self.config.args.single_threaded = True | ||
|
||
@classmethod | ||
def pre_init_hook(cls): | ||
"""A hook called before the task is initialized.""" | ||
log_to_stderr(logger) | ||
|
||
def _iterate_selected_nodes(self): | ||
nodes = sorted(self.select_nodes()) | ||
if not nodes: | ||
logger.warning('No nodes selected!') | ||
return | ||
for node in nodes: | ||
yield self.manifest.nodes[node] | ||
|
||
def generate_selectors(self): | ||
for node in self._iterate_selected_nodes(): | ||
if node.resource_type == NodeType.Source: | ||
yield 'source:{}'.format(node.unique_id) | ||
else: | ||
yield node.unique_id | ||
|
||
def generate_names(self): | ||
for node in self._iterate_selected_nodes(): | ||
if node.resource_type == NodeType.Source: | ||
yield '{0.source_name}.{0.name}'.format(node) | ||
else: | ||
yield node.name | ||
|
||
def generate_json(self): | ||
for node in self._iterate_selected_nodes(): | ||
yield json.dumps({ | ||
k: v | ||
for k, v in node.serialize().items() | ||
if k in self.ALLOWED_KEYS | ||
}) | ||
|
||
def generate_paths(self): | ||
for node in self._iterate_selected_nodes(): | ||
yield node.get('original_file_path') | ||
|
||
def run(self): | ||
ManifestTask._runtime_initialize(self) | ||
output = self.config.args.output | ||
if output == 'selector': | ||
generator = self.generate_selectors | ||
elif output == 'name': | ||
generator = self.generate_names | ||
elif output == 'json': | ||
generator = self.generate_json | ||
elif output == 'path': | ||
generator = self.generate_paths | ||
else: | ||
raise dbt.exceptions.IternalException( | ||
'Invalid output {}'.format(output) | ||
) | ||
for result in generator(): | ||
self.node_results.append(result) | ||
print(result) | ||
return self.node_results | ||
|
||
@property | ||
def resource_types(self): | ||
values = set(self.config.args.resource_types) | ||
if not values: | ||
return list(self.DEFAULT_RESOURCE_VALUES) | ||
|
||
if 'default' in values: | ||
values.remove('default') | ||
values.update(self.DEFAULT_RESOURCE_VALUES) | ||
if 'all' in values: | ||
values.remove('all') | ||
values.update(self.ALL_RESOURCE_VALUES) | ||
return list(values) | ||
|
||
def build_query(self): | ||
return { | ||
"include": self.args.models, | ||
"exclude": self.args.exclude, | ||
"resource_types": self.resource_types, | ||
"tags": [], | ||
} | ||
|
||
def interpret_results(self, results): | ||
return bool(results) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
select 4 as id |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
{% archive my_archive %} | ||
{{ | ||
config( | ||
target_database=var('target_database', database), | ||
target_schema=schema, | ||
unique_key='id', | ||
strategy='timestamp', | ||
updated_at='updated_at', | ||
) | ||
}} | ||
select * from {{database}}.{{schema}}.seed | ||
{% endarchive %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
a,b | ||
1,2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
{% macro cool_macro() %} | ||
wow! | ||
{% endmacro %} | ||
|
||
{% macro other_cool_macro(a, b) %} | ||
cool! | ||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{% docs my_docs %} | ||
some docs | ||
{% enddocs %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
select 1 as id |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
version: 2 | ||
models: | ||
- name: outer | ||
description: The outer table | ||
columns: | ||
- name: id | ||
description: The id value | ||
tests: | ||
- unique | ||
- not_null | ||
|
||
sources: | ||
- name: my_source | ||
tables: | ||
- name: my_table |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
select * from {{ ref('outer') }} |
Oops, something went wrong.