diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 4c8e21d4183..fdf8adedcfe 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -9,7 +9,7 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.utils import coalesce from dbt.node_types import NodeType -import dbt.exceptions +from dbt.exceptions import RuntimeException, InternalException, warn_or_error SELECTOR_PARENTS = '+' SELECTOR_CHILDREN = '+' @@ -50,7 +50,7 @@ def __init__(self, node_spec: str): node_spec = node_spec[:-1] if self.select_children and self.select_childrens_parents: - raise dbt.exceptions.RuntimeException( + raise RuntimeException( 'Invalid node spec {} - "@" prefix and "+" suffix are ' 'incompatible'.format(self.raw) ) @@ -79,6 +79,14 @@ def __str__(self): return self._value_ +def alert_non_existence(raw_spec, nodes): + if len(nodes) == 0: + warn_or_error( + f"The selector '{str(raw_spec)}' does not match any nodes and will" + f" be ignored" + ) + + def split_specs(node_specs: Iterable[str]): specs: Set[str] = set() for spec in node_specs: @@ -88,8 +96,8 @@ def split_specs(node_specs: Iterable[str]): return specs -def get_package_names(graph): - return set([node.split(".")[1] for node in graph.nodes()]) +def get_package_names(nodes): + return set([node.split(".")[1] for node in nodes]) def is_selected_node(real_node, node_selector): @@ -190,7 +198,7 @@ def search(self, included_nodes, selector): :param str selector: The selector or node name """ qualified_name = selector.split(".") - package_names = {node.split(".")[1] for node in included_nodes} + package_names = get_package_names(included_nodes) for node, real_node in self.parsed_nodes(included_nodes): if _node_is_match(qualified_name, package_names, real_node.fqn): yield node @@ -228,7 +236,7 @@ def search(self, included_nodes, selector): '`${{source_name}}.${{target_name}}`, or ' '`${{package_name}}.${{source_name}}.${{target_name}}' ).format(selector) - raise dbt.exceptions.RuntimeException(msg) + raise RuntimeException(msg) for node, real_node in self.source_nodes(included_nodes): if target_package not in (real_node.package_name, SELECTOR_GLOB): @@ -380,18 +388,47 @@ def get_nodes_from_spec(self, graph, spec): return collected - def select_nodes(self, graph, raw_include_specs, raw_exclude_specs): - selected_nodes: Set[str] = set() + def get_nodes_from_multiple_specs( + self, + graph, + specs, + nodes=None, + check_existence=False, + exclude=False + ): + selected_nodes: Set[str] = coalesce(nodes, set()) + operator = set.difference_update if exclude else set.update - for raw_spec in split_specs(raw_include_specs): + for raw_spec in split_specs(specs): spec = SelectionCriteria(raw_spec) - included_nodes = self.get_nodes_from_spec(graph, spec) - selected_nodes.update(included_nodes) + nodes = self.get_nodes_from_spec(graph, spec) - for raw_spec in split_specs(raw_exclude_specs): - spec = SelectionCriteria(raw_spec) - excluded_nodes = self.get_nodes_from_spec(graph, spec) - selected_nodes.difference_update(excluded_nodes) + if check_existence: + alert_non_existence(raw_spec, nodes) + + operator(selected_nodes, nodes) + + return selected_nodes + + def select_nodes(self, graph, raw_include_specs, raw_exclude_specs): + raw_exclude_specs = coalesce(raw_exclude_specs, []) + check_existance = True + + if not raw_include_specs: + check_existance = False + raw_include_specs = ['fqn:*', 'source:*'] + + selected_nodes = self.get_nodes_from_multiple_specs( + graph, + raw_include_specs, + check_existence=check_existance + ) + selected_nodes = self.get_nodes_from_multiple_specs( + graph, + raw_exclude_specs, + nodes=selected_nodes, + exclude=True + ) return selected_nodes @@ -407,7 +444,7 @@ def _is_match(self, node_name, resource_types, tags, required): elif node_name in self.manifest.sources: node = self.manifest.sources[node_name] else: - raise dbt.exceptions.InternalException( + raise InternalException( f'Node {node_name} not found in the manifest!' ) if node.resource_type not in resource_types: @@ -422,8 +459,6 @@ def _is_match(self, node_name, resource_types, tags, required): return True def get_selected(self, include, exclude, resource_types, tags, required): - include = coalesce(include, ['fqn:*', 'source:*']) - exclude = coalesce(exclude, []) tags = coalesce(tags, []) graph_members = { diff --git a/test/unit/test_graph_selection.py b/test/unit/test_graph_selection.py index 5e214c59925..f496b9a9ef5 100644 --- a/test/unit/test_graph_selection.py +++ b/test/unit/test_graph_selection.py @@ -173,7 +173,7 @@ def test__spec_parsing(self): self.invalid_spec('@source:a+') def test__package_name_getter(self): - found = graph_selector.get_package_names(self.package_graph) + found = graph_selector.get_package_names(self.package_graph.nodes()) expected = set(['X', 'Y']) self.assertEqual(found, expected)