diff --git a/CHANGELOG.md b/CHANGELOG.md index eb9a46ab9b0..463dc42c407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Customize ls task JSON output by adding new flag `--output-keys` ([#3778](https://github.com/dbt-labs/dbt/issues/3778), [#3395](https://github.com/dbt-labs/dbt/issues/3395)) - Add support for execution project on BigQuery through profile configuration ([#3707](https://github.com/dbt-labs/dbt/issues/3707), [#3708](https://github.com/dbt-labs/dbt/issues/3708)) - Skip downstream nodes during the `build` task when a test fails. ([#3597](https://github.com/dbt-labs/dbt/issues/3597), [#3792](https://github.com/dbt-labs/dbt/pull/3792)) -- Added default field in the `selectors.yml` to allow user to define default selector ([#3448](https://github.com/dbt-labs/dbt/issues/3448#issuecomment-907611470)) +- Added default field in the `selectors.yml` to allow user to define default selector ([#3448](https://github.com/dbt-labs/dbt/issues/3448), [#3875](https://github.com/dbt-labs/dbt/issues/3875), [#3892](https://github.com/dbt-labs/dbt/issues/3892)) - Added timing and thread information to sources.json artifact ([#3804](https://github.com/dbt-labs/dbt/issues/3804), [#3894](https://github.com/dbt-labs/dbt/pull/3894)) ### Fixes diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 568bb7fb40c..0d50470a1ca 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -38,7 +38,6 @@ selector_data_from_root, SelectorConfig, ) -from dbt.logger import print_timestamped_line INVALID_VERSION_ERROR = """\ @@ -654,16 +653,14 @@ def get_selector(self, name: str) -> Union[SelectionSpec, bool]: ) return self.selectors[name]["definition"] - def get_default_selector(self) -> Union[SelectionSpec, bool, None]: + def get_default_selector_name(self) -> Union[str, None]: """This function fetch the default selector to use on `dbt run` (if any) :return: either a selector if default is set or None :rtype: Union[SelectionSpec, None] """ for selector_name, selector in self.selectors.items(): if selector["default"] is True: - name = selector_name - print_timestamped_line(f'Using default selector {name}') - return self.get_selector(name) + return selector_name return None diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index 571f11d237a..464a4aa8c44 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -4,7 +4,7 @@ from dbt.contracts.results import RunStatus, RunResult from dbt.exceptions import InternalException -from dbt.graph import ResourceTypeSelector, SelectionSpec, parse_difference +from dbt.graph import ResourceTypeSelector from dbt.logger import print_timestamped_line from dbt.node_types import NodeType @@ -37,16 +37,6 @@ class CompileTask(GraphRunnableTask): def raise_on_first_error(self): return True - def get_selection_spec(self) -> SelectionSpec: - default_selector = self.config.get_default_selector() - if self.args.selector_name: - spec = self.config.get_selector(self.args.selector_name) - elif not (self.args.select or self.args.exclude) and default_selector: - spec = default_selector - else: - spec = parse_difference(self.args.select, self.args.exclude) - return spec - def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise InternalException( diff --git a/core/dbt/task/freshness.py b/core/dbt/task/freshness.py index aec6c7d8667..a0000399057 100644 --- a/core/dbt/task/freshness.py +++ b/core/dbt/task/freshness.py @@ -19,7 +19,7 @@ from dbt.logger import print_timestamped_line from dbt.node_types import NodeType -from dbt.graph import ResourceTypeSelector, SelectionSpec, parse_difference +from dbt.graph import ResourceTypeSelector from dbt.contracts.graph.parsed import ParsedSourceDefinition @@ -136,22 +136,6 @@ def result_path(self): def raise_on_first_error(self): return False - def get_selection_spec(self) -> SelectionSpec: - """Generates a selection spec from task arguments to use when - processing graph. A SelectionSpec describes what nodes to select - when creating queue from graph of nodes. - """ - default_selector = self.config.get_default_selector() - if self.args.selector_name: - # use pre-defined selector (--selector) to create selection spec - spec = self.config.get_selector(self.args.selector_name) - elif not (self.args.select or self.args.exclude) and default_selector: - spec = default_selector - else: - # use --select and --exclude args to create selection spec - spec = parse_difference(self.args.select, self.args.exclude) - return spec - def get_node_selector(self): if self.manifest is None or self.graph is None: raise InternalException( diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 6561b342cee..85277e07b7e 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -4,11 +4,7 @@ ParsedExposure, ParsedSourceDefinition ) -from dbt.graph import ( - parse_difference, - ResourceTypeSelector, - SelectionSpec, -) +from dbt.graph import ResourceTypeSelector from dbt.task.runnable import GraphRunnableTask, ManifestTask from dbt.task.test import TestSelector from dbt.node_types import NodeType @@ -164,22 +160,14 @@ def resource_types(self): return list(values) @property - def selector(self): + def selection_arg(self): + # for backwards compatibility, list accepts both --models and --select, + # with slightly different behavior: --models implies --resource-type model if self.args.models: return self.args.models else: return self.args.select - def get_selection_spec(self) -> SelectionSpec: - default_selector = self.config.get_default_selector() - if self.args.selector_name: - spec = self.config.get_selector(self.args.selector_name) - elif not (self.args.select or self.args.exclude) and default_selector: - spec = default_selector - else: - spec = parse_difference(self.selector, self.args.exclude) - return spec - def get_node_selector(self): if self.manifest is None or self.graph is None: raise InternalException( diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index df26fc8fdd7..c9ebb465b05 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -41,7 +41,13 @@ FailFastException, ) -from dbt.graph import GraphQueue, NodeSelector, SelectionSpec, Graph +from dbt.graph import ( + GraphQueue, + NodeSelector, + SelectionSpec, + parse_difference, + Graph +) from dbt.parser.manifest import ManifestLoader import dbt.exceptions @@ -106,11 +112,27 @@ def set_previous_state(self): def index_offset(self, value: int) -> int: return value - @abstractmethod + @property + def selection_arg(self): + return self.args.select + + @property + def exclusion_arg(self): + return self.args.exclude + def get_selection_spec(self) -> SelectionSpec: - raise NotImplementedException( - f'get_selection_spec not implemented for task {type(self)}' - ) + default_selector_name = self.config.get_default_selector_name() + if self.args.selector_name: + # use pre-defined selector (--selector) + spec = self.config.get_selector(self.args.selector_name) + elif not (self.selection_arg or self.exclusion_arg) and default_selector_name: + # use pre-defined selector (--selector) with default: true + logger.info(f"Using default selector {default_selector_name}") + spec = self.config.get_selector(default_selector_name) + else: + # use --select and --exclude args + spec = parse_difference(self.selection_arg, self.exclusion_arg) + return spec @abstractmethod def get_node_selector(self) -> NodeSelector: