Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix logging for default selector #3892

Merged
merged 3 commits into from
Sep 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
- Customize ls task JSON output by adding new flag `--output-keys` ([#3778](https:/dbt-labs/dbt/issues/3778), [#3395](https:/dbt-labs/dbt/issues/3395))
- Add support for execution project on BigQuery through profile configuration ([#3707](https:/dbt-labs/dbt/issues/3707), [#3708](https:/dbt-labs/dbt/issues/3708))
- Skip downstream nodes during the `build` task when a test fails. ([#3597](https:/dbt-labs/dbt/issues/3597), [#3792](https:/dbt-labs/dbt/pull/3792))
- Added default field in the `selectors.yml` to allow user to define default selector ([#3448](https:/dbt-labs/dbt/issues/3448#issuecomment-907611470))
- Added default field in the `selectors.yml` to allow user to define default selector ([#3448](https:/dbt-labs/dbt/issues/3448), [#3875](https:/dbt-labs/dbt/issues/3875), [#3892](https:/dbt-labs/dbt/issues/3892))
- Added timing and thread information to sources.json artifact ([#3804](https:/dbt-labs/dbt/issues/3804), [#3894](https:/dbt-labs/dbt/pull/3894))

### Fixes
Expand Down
7 changes: 2 additions & 5 deletions core/dbt/config/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
selector_data_from_root,
SelectorConfig,
)
from dbt.logger import print_timestamped_line


INVALID_VERSION_ERROR = """\
Expand Down Expand Up @@ -654,16 +653,14 @@ def get_selector(self, name: str) -> Union[SelectionSpec, bool]:
)
return self.selectors[name]["definition"]

def get_default_selector(self) -> Union[SelectionSpec, bool, None]:
def get_default_selector_name(self) -> Union[str, None]:
"""This function fetch the default selector to use on `dbt run` (if any)
:return: either a selector if default is set or None
:rtype: Union[SelectionSpec, None]
"""
for selector_name, selector in self.selectors.items():
if selector["default"] is True:
name = selector_name
print_timestamped_line(f'Using default selector {name}')
return self.get_selector(name)
return selector_name

return None

Expand Down
12 changes: 1 addition & 11 deletions core/dbt/task/compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from dbt.contracts.results import RunStatus, RunResult
from dbt.exceptions import InternalException
from dbt.graph import ResourceTypeSelector, SelectionSpec, parse_difference
from dbt.graph import ResourceTypeSelector
from dbt.logger import print_timestamped_line
from dbt.node_types import NodeType

Expand Down Expand Up @@ -37,16 +37,6 @@ class CompileTask(GraphRunnableTask):
def raise_on_first_error(self):
return True

def get_selection_spec(self) -> SelectionSpec:
default_selector = self.config.get_default_selector()
if self.args.selector_name:
spec = self.config.get_selector(self.args.selector_name)
elif not (self.args.select or self.args.exclude) and default_selector:
spec = default_selector
else:
spec = parse_difference(self.args.select, self.args.exclude)
return spec

def get_node_selector(self) -> ResourceTypeSelector:
if self.manifest is None or self.graph is None:
raise InternalException(
Expand Down
18 changes: 1 addition & 17 deletions core/dbt/task/freshness.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from dbt.logger import print_timestamped_line
from dbt.node_types import NodeType

from dbt.graph import ResourceTypeSelector, SelectionSpec, parse_difference
from dbt.graph import ResourceTypeSelector
from dbt.contracts.graph.parsed import ParsedSourceDefinition


Expand Down Expand Up @@ -136,22 +136,6 @@ def result_path(self):
def raise_on_first_error(self):
return False

def get_selection_spec(self) -> SelectionSpec:
"""Generates a selection spec from task arguments to use when
processing graph. A SelectionSpec describes what nodes to select
when creating queue from graph of nodes.
"""
default_selector = self.config.get_default_selector()
if self.args.selector_name:
# use pre-defined selector (--selector) to create selection spec
spec = self.config.get_selector(self.args.selector_name)
elif not (self.args.select or self.args.exclude) and default_selector:
spec = default_selector
else:
# use --select and --exclude args to create selection spec
spec = parse_difference(self.args.select, self.args.exclude)
return spec

def get_node_selector(self):
if self.manifest is None or self.graph is None:
raise InternalException(
Expand Down
20 changes: 4 additions & 16 deletions core/dbt/task/list.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,7 @@
ParsedExposure,
ParsedSourceDefinition
)
from dbt.graph import (
parse_difference,
ResourceTypeSelector,
SelectionSpec,
)
from dbt.graph import ResourceTypeSelector
from dbt.task.runnable import GraphRunnableTask, ManifestTask
from dbt.task.test import TestSelector
from dbt.node_types import NodeType
Expand Down Expand Up @@ -164,22 +160,14 @@ def resource_types(self):
return list(values)

@property
def selector(self):
def selection_arg(self):
# for backwards compatibility, list accepts both --models and --select,
# with slightly different behavior: --models implies --resource-type model
if self.args.models:
return self.args.models
else:
return self.args.select

def get_selection_spec(self) -> SelectionSpec:
default_selector = self.config.get_default_selector()
if self.args.selector_name:
spec = self.config.get_selector(self.args.selector_name)
elif not (self.args.select or self.args.exclude) and default_selector:
spec = default_selector
else:
spec = parse_difference(self.selector, self.args.exclude)
return spec

def get_node_selector(self):
if self.manifest is None or self.graph is None:
raise InternalException(
Expand Down
32 changes: 27 additions & 5 deletions core/dbt/task/runnable.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,13 @@
FailFastException,
)

from dbt.graph import GraphQueue, NodeSelector, SelectionSpec, Graph
from dbt.graph import (
GraphQueue,
NodeSelector,
SelectionSpec,
parse_difference,
Graph
)
from dbt.parser.manifest import ManifestLoader

import dbt.exceptions
Expand Down Expand Up @@ -106,11 +112,27 @@ def set_previous_state(self):
def index_offset(self, value: int) -> int:
return value

@abstractmethod
@property
def selection_arg(self):
return self.args.select

@property
def exclusion_arg(self):
return self.args.exclude

def get_selection_spec(self) -> SelectionSpec:
raise NotImplementedException(
f'get_selection_spec not implemented for task {type(self)}'
)
default_selector_name = self.config.get_default_selector_name()
if self.args.selector_name:
# use pre-defined selector (--selector)
spec = self.config.get_selector(self.args.selector_name)
elif not (self.selection_arg or self.exclusion_arg) and default_selector_name:
# use pre-defined selector (--selector) with default: true
logger.info(f"Using default selector {default_selector_name}")
spec = self.config.get_selector(default_selector_name)
else:
# use --select and --exclude args
spec = parse_difference(self.selection_arg, self.exclusion_arg)
return spec

@abstractmethod
def get_node_selector(self) -> NodeSelector:
Expand Down