diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 51db9d072de..942195d664a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.18.0 +current_version = 0.18.1a1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c34522949..32d8ebda4fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https://github.com/fishtown-analytics/dbt/pull/2680), [#2723](https://github.com/fishtown-analytics/dbt/pull/2723)) +- Added "reports" ([#2730](https://github.com/fishtown-analytics/dbt/issues/2730), [#2752](https://github.com/fishtown-analytics/dbt/pull/2752)) Contributors: - [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723)) diff --git a/core/dbt/adapters/protocol.py b/core/dbt/adapters/protocol.py index ab5cff64f41..b9598bd0a75 100644 --- a/core/dbt/adapters/protocol.py +++ b/core/dbt/adapters/protocol.py @@ -9,7 +9,7 @@ from dbt.contracts.connection import Connection, AdapterRequiredConfig from dbt.contracts.graph.compiled import ( - CompiledNode, NonSourceNode, NonSourceCompiledNode + CompiledNode, ManifestNode, NonSourceCompiledNode ) from dbt.contracts.graph.parsed import ParsedNode, ParsedSourceDefinition from dbt.contracts.graph.model_config import BaseConfig @@ -55,7 +55,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph: def compile_node( self, - node: NonSourceNode, + node: ManifestNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]] = None, ) -> NonSourceCompiledNode: diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index e8713d686a4..5811227efe4 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -12,12 +12,13 @@ from dbt.context.providers import generate_runtime_model from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.compiled import ( - InjectedCTE, - COMPILED_TYPES, - NonSourceNode, - NonSourceCompiledNode, CompiledDataTestNode, CompiledSchemaTestNode, + COMPILED_TYPES, + GraphMemberNode, + InjectedCTE, + ManifestNode, + NonSourceCompiledNode, ) from dbt.contracts.graph.parsed import ParsedNode from dbt.exceptions import ( @@ -64,7 +65,7 @@ def print_compile_stats(stats): logger.info("Found {}".format(stat_line)) -def _node_enabled(node: NonSourceNode): +def _node_enabled(node: ManifestNode): # Disabled models are already excluded from the manifest if node.resource_type == NodeType.Test and not node.config.enabled: return False @@ -358,7 +359,7 @@ def _insert_ctes( def _compile_node( self, - node: NonSourceNode, + node: ManifestNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]] = None, ) -> NonSourceCompiledNode: @@ -402,7 +403,7 @@ def write_graph_file(self, linker: Linker, manifest: Manifest): linker.write_graph(graph_path, manifest) def link_node( - self, linker: Linker, node: NonSourceNode, manifest: Manifest + self, linker: Linker, node: GraphMemberNode, manifest: Manifest ): linker.add_node(node.unique_id) @@ -425,6 +426,9 @@ def link_graph(self, linker: Linker, manifest: Manifest): linker.add_node(source.unique_id) for node in manifest.nodes.values(): self.link_node(linker, node, manifest) + for report in manifest.reports.values(): + self.link_node(linker, report, manifest) + # linker.add_node(report.unique_id) cycle = linker.find_cycles() @@ -445,7 +449,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph: return Graph(linker.graph) - def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode: + def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode: if not _is_writable(node): return node logger.debug(f'Writing injected SQL for node "{node.unique_id}"') @@ -467,7 +471,7 @@ def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode: def compile_node( self, - node: NonSourceNode, + node: ManifestNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]] = None, write: bool = True, diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 16d79213f30..e2d9fca2035 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -21,10 +21,11 @@ from dbt.contracts.graph.compiled import ( CompiledResource, CompiledSeedNode, - NonSourceNode, + ManifestNode, ) from dbt.contracts.graph.parsed import ( ParsedMacro, + ParsedReport, ParsedSeedNode, ParsedSourceDefinition, ) @@ -419,7 +420,7 @@ def resolve( return self.Relation.create_from(self.config, self.model) -ResolveRef = Union[Disabled, NonSourceNode] +ResolveRef = Union[Disabled, ManifestNode] class RuntimeRefResolver(BaseRefResolver): @@ -444,7 +445,7 @@ def resolve( return self.create_relation(target_model, target_name) def create_relation( - self, target_model: NonSourceNode, name: str + self, target_model: ManifestNode, name: str ) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) @@ -456,7 +457,7 @@ def create_relation( def validate( self, - resolved: NonSourceNode, + resolved: ManifestNode, target_name: str, target_package: Optional[str] ) -> None: @@ -468,14 +469,14 @@ def validate( class OperationRefResolver(RuntimeRefResolver): def validate( self, - resolved: NonSourceNode, + resolved: ManifestNode, target_name: str, target_package: Optional[str], ) -> None: pass def create_relation( - self, target_model: NonSourceNode, name: str + self, target_model: ManifestNode, name: str ) -> RelationProxy: if target_model.is_ephemeral_model: # In operations, we can't ref() ephemeral nodes, because @@ -627,7 +628,7 @@ def __init__( ) # mypy appeasement - we know it'll be a RuntimeConfig self.config: RuntimeConfig - self.model: Union[ParsedMacro, NonSourceNode] = model + self.model: Union[ParsedMacro, ManifestNode] = model super().__init__(config, manifest, model.package_name) self.sql_results: Dict[str, AttrDict] = {} self.context_config: Optional[ContextConfigType] = context_config @@ -1196,7 +1197,7 @@ def __init__( class ModelContext(ProviderContext): - model: NonSourceNode + model: ManifestNode @contextproperty def pre_hooks(self) -> List[Dict[str, Any]]: @@ -1267,7 +1268,7 @@ def this(self) -> Optional[RelationProxy]: def generate_parser_model( - model: NonSourceNode, + model: ManifestNode, config: RuntimeConfig, manifest: Manifest, context_config: ContextConfigType, @@ -1302,7 +1303,7 @@ def generate_generate_component_name_macro( def generate_runtime_model( - model: NonSourceNode, + model: ManifestNode, config: RuntimeConfig, manifest: Manifest, ) -> Dict[str, Any]: @@ -1322,3 +1323,45 @@ def generate_runtime_macro( macro, config, manifest, OperationProvider(), package_name ) return ctx.to_dict() + + +class ReportRefResolver(BaseResolver): + def __call__(self, *args) -> str: + if len(args) not in (1, 2): + ref_invalid_args(self.model, args) + self.model.refs.append(list(args)) + return '' + + +class ReportSourceResolver(BaseResolver): + def __call__(self, *args) -> str: + if len(args) != 2: + raise_compiler_error( + f"source() takes exactly two arguments ({len(args)} given)", + self.model + ) + self.model.sources.append(list(args)) + return '' + + +def generate_parse_report( + report: ParsedReport, + config: RuntimeConfig, + manifest: Manifest, + package_name: str, +) -> Dict[str, Any]: + project = config.load_dependencies()[package_name] + return { + 'ref': ReportRefResolver( + None, + report, + project, + manifest, + ), + 'source': ReportSourceResolver( + None, + report, + project, + manifest, + ) + } diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index fc36fbaebb5..4ef0bab54cc 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -121,6 +121,7 @@ class SourceFile(JsonSchemaMixin): docs: List[str] = field(default_factory=list) macros: List[str] = field(default_factory=list) sources: List[str] = field(default_factory=list) + reports: List[str] = field(default_factory=list) # any node patches in this file. The entries are names, not unique ids! patches: List[str] = field(default_factory=list) # any macro patches in this file. The entries are package, name pairs. diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index 1406475f969..2eeae63861b 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -5,6 +5,7 @@ ParsedDataTestNode, ParsedHookNode, ParsedModelNode, + ParsedReport, ParsedResource, ParsedRPCNode, ParsedSchemaTestNode, @@ -202,7 +203,7 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource: # This is anything that can be in manifest.nodes. -NonSourceNode = Union[ +ManifestNode = Union[ NonSourceCompiledNode, NonSourceParsedNode, ] @@ -211,6 +212,12 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource: # 'compile()' calls in the runner actually just return the original parsed # node they were given. CompileResultNode = Union[ - NonSourceNode, + ManifestNode, ParsedSourceDefinition, ] + +# anything that participates in the graph: sources, reports, manifest nodes +GraphMemberNode = Union[ + CompileResultNode, + ParsedReport, +] diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index f6a4b10beea..f2f5f476a44 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -14,11 +14,11 @@ from hologram import JsonSchemaMixin from dbt.contracts.graph.compiled import ( - CompileResultNode, NonSourceNode, NonSourceCompiledNode + CompileResultNode, ManifestNode, NonSourceCompiledNode, GraphMemberNode ) from dbt.contracts.graph.parsed import ( ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch, - ParsedSourceDefinition + ParsedSourceDefinition, ParsedReport ) from dbt.contracts.files import SourceFile from dbt.contracts.util import ( @@ -130,7 +130,7 @@ def perform_lookup( return self._manifest.sources[unique_id] -class RefableCache(PackageAwareCache[RefName, NonSourceNode]): +class RefableCache(PackageAwareCache[RefName, ManifestNode]): # refables are actually unique, so the Dict[PackageName, UniqueID] will # only ever have exactly one value, but doing 3 dict lookups instead of 1 # is not a big deal at all and retains consistency @@ -138,7 +138,7 @@ def __init__(self, manifest: 'Manifest'): self._cached_types = set(NodeType.refable()) super().__init__(manifest) - def add_node(self, node: NonSourceNode): + def add_node(self, node: ManifestNode): if node.resource_type in self._cached_types: if node.name not in self.storage: self.storage[node.name] = {} @@ -150,7 +150,7 @@ def populate(self): def perform_lookup( self, unique_id: UniqueID - ) -> NonSourceNode: + ) -> ManifestNode: if unique_id not in self._manifest.nodes: raise dbt.exceptions.InternalException( f'Node {unique_id} found in cache but not found in manifest' @@ -217,7 +217,7 @@ def _sort_values(dct): return {k: sorted(v) for k, v in dct.items()} -def build_edges(nodes: List[NonSourceNode]): +def build_edges(nodes: List[ManifestNode]): """Build the forward and backward edges on the given list of ParsedNodes and return them as two separate dictionaries, each mapping unique IDs to lists of edges. @@ -393,12 +393,12 @@ class Disabled(Generic[D]): MaybeNonSource = Optional[Union[ - NonSourceNode, - Disabled[NonSourceNode] + ManifestNode, + Disabled[ManifestNode] ]] -T = TypeVar('T', bound=CompileResultNode) +T = TypeVar('T', bound=GraphMemberNode) def _update_into(dest: MutableMapping[str, T], new_item: T): @@ -425,10 +425,11 @@ def _update_into(dest: MutableMapping[str, T], new_item: T): class Manifest: """The manifest for the full graph, after parsing and during compilation. """ - nodes: MutableMapping[str, NonSourceNode] + nodes: MutableMapping[str, ManifestNode] sources: MutableMapping[str, ParsedSourceDefinition] macros: MutableMapping[str, ParsedMacro] docs: MutableMapping[str, ParsedDocumentation] + reports: MutableMapping[str, ParsedReport] generated_at: datetime disabled: List[CompileResultNode] files: MutableMapping[str, SourceFile] @@ -454,6 +455,7 @@ def from_macros( sources={}, macros=macros, docs={}, + reports={}, generated_at=datetime.utcnow(), disabled=[], files=files, @@ -479,7 +481,10 @@ def sync_update_node( _update_into(self.nodes, new_node) return new_node - def update_node(self, new_node: NonSourceNode): + def update_report(self, new_report: ParsedReport): + _update_into(self.reports, new_report) + + def update_node(self, new_node: ManifestNode): _update_into(self.nodes, new_node) def update_source(self, new_source: ParsedSourceDefinition): @@ -502,7 +507,7 @@ def build_flat_graph(self): def find_disabled_by_name( self, name: str, package: Optional[str] = None - ) -> Optional[NonSourceNode]: + ) -> Optional[ManifestNode]: searcher: NameSearcher = NameSearcher( name, package, NodeType.refable() ) @@ -632,7 +637,7 @@ def get_resource_fqns(self) -> Mapping[str, PathSet]: resource_fqns[resource_type_plural].add(tuple(resource.fqn)) return resource_fqns - def add_nodes(self, new_nodes: Mapping[str, NonSourceNode]): + def add_nodes(self, new_nodes: Mapping[str, ManifestNode]): """Add the given dict of new nodes to the manifest.""" for unique_id, node in new_nodes.items(): if unique_id in self.nodes: @@ -720,6 +725,7 @@ def deepcopy(self): sources={k: _deepcopy(v) for k, v in self.sources.items()}, macros={k: _deepcopy(v) for k, v in self.macros.items()}, docs={k: _deepcopy(v) for k, v in self.docs.items()}, + reports={k: _deepcopy(v) for k, v in self.reports.items()}, generated_at=self.generated_at, disabled=[_deepcopy(n) for n in self.disabled], metadata=self.metadata, @@ -727,7 +733,11 @@ def deepcopy(self): ) def writable_manifest(self): - edge_members = list(chain(self.nodes.values(), self.sources.values())) + edge_members = list(chain( + self.nodes.values(), + self.sources.values(), + self.reports.values(), + )) forward_edges, backward_edges = build_edges(edge_members) return WritableManifest( @@ -735,6 +745,7 @@ def writable_manifest(self): sources=self.sources, macros=self.macros, docs=self.docs, + reports=self.reports, generated_at=self.generated_at, metadata=self.metadata, disabled=self.disabled, @@ -750,11 +761,13 @@ def to_dict(self, omit_none=True, validate=False): def write(self, path): self.writable_manifest().write(path) - def expect(self, unique_id: str) -> CompileResultNode: + def expect(self, unique_id: str) -> GraphMemberNode: if unique_id in self.nodes: return self.nodes[unique_id] elif unique_id in self.sources: return self.sources[unique_id] + elif unique_id in self.reports: + return self.reports[unique_id] else: # something terrible has happened raise dbt.exceptions.InternalException( @@ -793,8 +806,8 @@ def resolve_ref( node_package: str, ) -> MaybeNonSource: - node: Optional[NonSourceNode] = None - disabled: Optional[NonSourceNode] = None + node: Optional[ManifestNode] = None + disabled: Optional[ManifestNode] = None candidates = _search_packages( current_project, node_package, target_model_package @@ -897,6 +910,7 @@ def __reduce_ex__(self, protocol): self.sources, self.macros, self.docs, + self.reports, self.generated_at, self.disabled, self.files, @@ -911,7 +925,7 @@ def __reduce_ex__(self, protocol): @dataclass class WritableManifest(JsonSchemaMixin, Writable, Readable): - nodes: Mapping[UniqueID, NonSourceNode] = field( + nodes: Mapping[UniqueID, ManifestNode] = field( metadata=dict(description=( 'The nodes defined in the dbt project and its dependencies' )) @@ -931,6 +945,11 @@ class WritableManifest(JsonSchemaMixin, Writable, Readable): 'The docs defined in the dbt project and its dependencies' )) ) + reports: Mapping[UniqueID, ParsedReport] = field( + metadata=dict(description=( + 'The reports defined in the dbt project and its dependencies' + )) + ) disabled: Optional[List[CompileResultNode]] = field(metadata=dict( description='A list of the disabled nodes in the target' )) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 2b3e6f74ee8..dccf90d53cf 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -22,7 +22,8 @@ UnparsedNode, UnparsedDocumentation, Quoting, Docs, UnparsedBaseNode, FreshnessThreshold, ExternalTable, HasYamlMetadata, MacroArgument, UnparsedSourceDefinition, - UnparsedSourceTableDefinition, UnparsedColumn, TestDef + UnparsedSourceTableDefinition, UnparsedColumn, TestDef, + ReportOwner, ExposureType, MaturityType ) from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin from dbt.exceptions import warn_or_error @@ -634,6 +635,71 @@ def search_name(self): return f'{self.source_name}.{self.name}' +@dataclass +class ParsedReport(UnparsedBaseNode, HasUniqueID, HasFqn): + name: str + type: ExposureType + owner: ReportOwner + resource_type: NodeType = NodeType.Report + maturity: Optional[MaturityType] = None + url: Optional[str] = None + description: Optional[str] = None + depends_on: DependsOn = field(default_factory=DependsOn) + refs: List[List[str]] = field(default_factory=list) + sources: List[List[str]] = field(default_factory=list) + + @property + def depends_on_nodes(self): + return self.depends_on.nodes + + @property + def search_name(self): + return self.name + + # no tags for now, but we could definitely add them + @property + def tags(self): + return [] + + def same_depends_on(self, old: 'ParsedReport') -> bool: + return set(self.depends_on.nodes) == set(old.depends_on.nodes) + + def same_description(self, old: 'ParsedReport') -> bool: + return self.description == old.description + + def same_maturity(self, old: 'ParsedReport') -> bool: + return self.maturity == old.maturity + + def same_owner(self, old: 'ParsedReport') -> bool: + return self.owner == old.owner + + def same_exposure_type(self, old: 'ParsedReport') -> bool: + return self.type == old.type + + def same_url(self, old: 'ParsedReport') -> bool: + return self.url == old.url + + def same_contents(self, old: Optional['ParsedReport']) -> bool: + # existing when it didn't before is a change! + if old is None: + return True + + return ( + self.same_fqn(old) and + self.same_exposure_type(old) and + self.same_owner(old) and + self.same_maturity(old) and + self.same_url(old) and + self.same_description(old) and + self.same_depends_on(old) and + True + ) + + ParsedResource = Union[ - ParsedMacro, ParsedNode, ParsedDocumentation, ParsedSourceDefinition + ParsedDocumentation, + ParsedMacro, + ParsedNode, + ParsedReport, + ParsedSourceDefinition, ] diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 105b41a2712..b14f5fe2202 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -359,3 +359,63 @@ def resource_type(self): @dataclass class UnparsedDocumentationFile(UnparsedDocumentation): file_contents: str + + +# can't use total_ordering decorator here, as str provides an ordering already +# and it's not the one we want. +class Maturity(StrEnum): + low = 'low' + medium = 'medium' + high = 'high' + + def __lt__(self, other): + if not isinstance(other, Maturity): + return NotImplemented + order = (Maturity.low, Maturity.medium, Maturity.high) + return order.index(self) < order.index(other) + + def __gt__(self, other): + if not isinstance(other, Maturity): + return NotImplemented + return self != other and not (self < other) + + def __ge__(self, other): + if not isinstance(other, Maturity): + return NotImplemented + return self == other or not (self < other) + + def __le__(self, other): + if not isinstance(other, Maturity): + return NotImplemented + return self == other or self < other + + +class ExposureType(StrEnum): + Dashboard = 'dashboard' + Notebook = 'notebook' + Analysis = 'analysis' + ML = 'ml' + Application = 'application' + + +class MaturityType(StrEnum): + Low = 'low' + Medium = 'medium' + High = 'high' + + +@dataclass +class ReportOwner(JsonSchemaMixin, Replaceable): + email: str + name: Optional[str] = None + + +@dataclass +class UnparsedReport(JsonSchemaMixin, Replaceable): + name: str + type: ExposureType + owner: ReportOwner + maturity: Optional[MaturityType] = None + url: Optional[str] = None + description: Optional[str] = None + depends_on: List[str] = field(default_factory=list) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 7eabb501902..b743022472c 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -18,7 +18,7 @@ INTERSECTION_DELIMITER = ',' -DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*'] +DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'report:*'] DEFAULT_EXCLUDES: List[str] = [] DATA_TEST_SELECTOR: str = 'test_type:data' SCHEMA_TEST_SELECTOR: str = 'test_type:schema' diff --git a/core/dbt/graph/queue.py b/core/dbt/graph/queue.py index 819781dd369..f08a6ad97a5 100644 --- a/core/dbt/graph/queue.py +++ b/core/dbt/graph/queue.py @@ -7,8 +7,8 @@ import networkx as nx # type: ignore from .graph import UniqueId -from dbt.contracts.graph.parsed import ParsedSourceDefinition -from dbt.contracts.graph.compiled import CompileResultNode +from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedReport +from dbt.contracts.graph.compiled import GraphMemberNode from dbt.contracts.graph.manifest import Manifest from dbt.node_types import NodeType @@ -50,8 +50,8 @@ def _include_in_cost(self, node_id: UniqueId) -> bool: node = self.manifest.expect(node_id) if node.resource_type != NodeType.Model: return False - # must be a Model - tell mypy this won't be a Source - assert not isinstance(node, ParsedSourceDefinition) + # must be a Model - tell mypy this won't be a Source or Report + assert not isinstance(node, (ParsedSourceDefinition, ParsedReport)) if node.is_ephemeral: return False return True @@ -84,7 +84,7 @@ def _calculate_scores(self) -> Dict[UniqueId, int]: def get( self, block: bool = True, timeout: Optional[float] = None - ) -> CompileResultNode: + ) -> GraphMemberNode: """Get a node off the inner priority queue. By default, this blocks. This takes the lock, but only for part of it. diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 4cb09ce9b42..ec687c0ca24 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -1,5 +1,5 @@ -from typing import Set, List, Union, Optional +from typing import Set, List, Optional from .graph import Graph, UniqueId from .queue import GraphQueue @@ -13,9 +13,8 @@ InvalidSelectorException, warn_or_error, ) -from dbt.contracts.graph.compiled import NonSourceNode, CompileResultNode +from dbt.contracts.graph.compiled import GraphMemberNode from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.parsed import ParsedSourceDefinition from dbt.contracts.state import PreviousState @@ -130,24 +129,25 @@ def _is_graph_member(self, unique_id: UniqueId) -> bool: if unique_id in self.manifest.sources: source = self.manifest.sources[unique_id] return source.config.enabled + elif unique_id in self.manifest.reports: + return True node = self.manifest.nodes[unique_id] return not node.empty and node.config.enabled - def node_is_match( - self, - node: Union[ParsedSourceDefinition, NonSourceNode], - ) -> bool: + def node_is_match(self, node: GraphMemberNode) -> bool: """Determine if a node is a match for the selector. Non-match nodes will be excluded from results during filtering. """ return True def _is_match(self, unique_id: UniqueId) -> bool: - node: CompileResultNode + node: GraphMemberNode if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] elif unique_id in self.manifest.sources: node = self.manifest.sources[unique_id] + elif unique_id in self.manifest.reports: + node = self.manifest.reports[unique_id] else: raise InternalException( f'Node {unique_id} not found in the manifest!' diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index aa289ab82f5..5ecac01c51a 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -10,12 +10,14 @@ from dbt.contracts.graph.compiled import ( CompiledDataTestNode, CompiledSchemaTestNode, - NonSourceNode, + CompileResultNode, + ManifestNode, ) from dbt.contracts.graph.manifest import Manifest, WritableManifest from dbt.contracts.graph.parsed import ( HasTestMetadata, ParsedDataTestNode, + ParsedReport, ParsedSchemaTestNode, ParsedSourceDefinition, ) @@ -44,6 +46,7 @@ class MethodName(StrEnum): TestType = 'test_type' ResourceType = 'resource_type' State = 'state' + Report = 'report' def is_selected_node(real_node, node_selector): @@ -72,7 +75,7 @@ def is_selected_node(real_node, node_selector): return True -SelectorTarget = Union[ParsedSourceDefinition, NonSourceNode] +SelectorTarget = Union[ParsedSourceDefinition, ManifestNode, ParsedReport] class SelectorMethod(metaclass=abc.ABCMeta): @@ -89,7 +92,7 @@ def __init__( def parsed_nodes( self, included_nodes: Set[UniqueId] - ) -> Iterator[Tuple[UniqueId, NonSourceNode]]: + ) -> Iterator[Tuple[UniqueId, ManifestNode]]: for key, node in self.manifest.nodes.items(): unique_id = UniqueId(key) @@ -108,13 +111,39 @@ def source_nodes( continue yield unique_id, source + def report_nodes( + self, + included_nodes: Set[UniqueId] + ) -> Iterator[Tuple[UniqueId, ParsedReport]]: + + for key, report in self.manifest.reports.items(): + unique_id = UniqueId(key) + if unique_id not in included_nodes: + continue + yield unique_id, report + def all_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, SelectorTarget]]: + yield from chain(self.parsed_nodes(included_nodes), + self.source_nodes(included_nodes), + self.report_nodes(included_nodes)) + + def configurable_nodes( + self, + included_nodes: Set[UniqueId] + ) -> Iterator[Tuple[UniqueId, CompileResultNode]]: yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes)) + def non_source_nodes( + self, + included_nodes: Set[UniqueId], + ) -> Iterator[Tuple[UniqueId, Union[ParsedReport, ManifestNode]]]: + yield from chain(self.parsed_nodes(included_nodes), + self.report_nodes(included_nodes)) + @abc.abstractmethod def search( self, @@ -209,8 +238,37 @@ def search( continue if target_source not in (real_node.source_name, SELECTOR_GLOB): continue - if target_table in (None, real_node.name, SELECTOR_GLOB): - yield node + if target_table not in (None, real_node.name, SELECTOR_GLOB): + continue + + yield node + + +class ReportSelectorMethod(SelectorMethod): + def search( + self, included_nodes: Set[UniqueId], selector: str + ) -> Iterator[UniqueId]: + parts = selector.split('.') + target_package = SELECTOR_GLOB + if len(parts) == 1: + target_name = parts[0] + elif len(parts) == 2: + target_package, target_name = parts + else: + msg = ( + 'Invalid report selector value "{}". Reports must be of ' + 'the form ${{report_name}} or ' + '${{report_package.report_name}}' + ).format(selector) + raise RuntimeException(msg) + + for node, real_node in self.report_nodes(included_nodes): + if target_package not in (real_node.package_name, SELECTOR_GLOB): + continue + if target_name not in (real_node.name, SELECTOR_GLOB): + continue + + yield node class PathSelectorMethod(SelectorMethod): @@ -284,7 +342,7 @@ def search( # search sources is kind of useless now source configs only have # 'enabled', which you can't really filter on anyway, but maybe we'll # add more someday, so search them anyway. - for node, real_node in self.all_nodes(included_nodes): + for node, real_node in self.configurable_nodes(included_nodes): try: value = _getattr_descend(real_node.config, parts) except AttributeError: @@ -423,6 +481,8 @@ def search( previous_node = manifest.nodes[node] elif node in manifest.sources: previous_node = manifest.sources[node] + elif node in manifest.reports: + previous_node = manifest.reports[node] if checker(previous_node, real_node): yield node @@ -439,6 +499,7 @@ class MethodManager: MethodName.TestName: TestNameSelectorMethod, MethodName.TestType: TestTypeSelectorMethod, MethodName.State: StateSelectorMethod, + MethodName.Report: ReportSelectorMethod, } def __init__( diff --git a/core/dbt/node_types.py b/core/dbt/node_types.py index 0eb3b60db03..1d14be2032f 100644 --- a/core/dbt/node_types.py +++ b/core/dbt/node_types.py @@ -14,6 +14,7 @@ class NodeType(StrEnum): Documentation = 'docs' Source = 'source' Macro = 'macro' + Report = 'report' @classmethod def executable(cls) -> List['NodeType']: diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 4e711c3e87d..3b353f6a5a9 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -20,10 +20,10 @@ from dbt.config import Project, RuntimeConfig from dbt.context.docs import generate_runtime_docs from dbt.contracts.files import FilePath, FileHash -from dbt.contracts.graph.compiled import NonSourceNode +from dbt.contracts.graph.compiled import ManifestNode from dbt.contracts.graph.manifest import Manifest, Disabled from dbt.contracts.graph.parsed import ( - ParsedSourceDefinition, ParsedNode, ParsedMacro, ColumnInfo, + ParsedSourceDefinition, ParsedNode, ParsedMacro, ColumnInfo, ParsedReport ) from dbt.exceptions import ( ref_target_not_found, @@ -307,7 +307,7 @@ def create_manifest(self) -> Manifest: for value in self.results.disabled.values(): disabled.extend(value) - nodes: MutableMapping[str, NonSourceNode] = { + nodes: MutableMapping[str, ManifestNode] = { k: v for k, v in self.results.nodes.items() } @@ -316,6 +316,7 @@ def create_manifest(self) -> Manifest: sources=sources, macros=self.results.macros, docs=self.results.docs, + reports=self.results.reports, generated_at=datetime.utcnow(), metadata=self.root_project.get_metadata(), disabled=disabled, @@ -414,8 +415,8 @@ def _check_resource_uniqueness( manifest: Manifest, config: RuntimeConfig, ) -> None: - names_resources: Dict[str, NonSourceNode] = {} - alias_resources: Dict[str, NonSourceNode] = {} + names_resources: Dict[str, ManifestNode] = {} + alias_resources: Dict[str, ManifestNode] = {} for resource, node in manifest.nodes.items(): if node.resource_type not in NodeType.refable(): @@ -493,7 +494,7 @@ def _get_node_column(node, column_name): def _process_docs_for_node( context: Dict[str, Any], - node: NonSourceNode, + node: ManifestNode, ): node.description = get_rendered(node.description, context) for column_name, column in node.columns.items(): @@ -552,12 +553,53 @@ def process_docs(manifest: Manifest, config: RuntimeConfig): _process_docs_for_macro(ctx, macro) +def _process_refs_for_report( + manifest: Manifest, current_project: str, report: ParsedReport +): + """Given a manifest and a report in that manifest, process its refs""" + for ref in report.refs: + target_model: Optional[Union[Disabled, ManifestNode]] = None + target_model_name: str + target_model_package: Optional[str] = None + + if len(ref) == 1: + target_model_name = ref[0] + elif len(ref) == 2: + target_model_package, target_model_name = ref + else: + raise dbt.exceptions.InternalException( + f'Refs should always be 1 or 2 arguments - got {len(ref)}' + ) + + target_model = manifest.resolve_ref( + target_model_name, + target_model_package, + current_project, + report.package_name, + ) + + if target_model is None or isinstance(target_model, Disabled): + # This may raise. Even if it doesn't, we don't want to add + # this report to the graph b/c there is no destination report + invalid_ref_fail_unless_test( + report, target_model_name, target_model_package, + disabled=(isinstance(target_model, Disabled)) + ) + + continue + + target_model_id = target_model.unique_id + + report.depends_on.nodes.append(target_model_id) + manifest.update_report(report) + + def _process_refs_for_node( - manifest: Manifest, current_project: str, node: NonSourceNode + manifest: Manifest, current_project: str, node: ManifestNode ): """Given a manifest and a node in that manifest, process its refs""" for ref in node.refs: - target_model: Optional[Union[Disabled, NonSourceNode]] = None + target_model: Optional[Union[Disabled, ManifestNode]] = None target_model_name: str target_model_package: Optional[str] = None @@ -600,11 +642,37 @@ def _process_refs_for_node( def process_refs(manifest: Manifest, current_project: str): for node in manifest.nodes.values(): _process_refs_for_node(manifest, current_project, node) + for report in manifest.reports.values(): + _process_refs_for_report(manifest, current_project, report) return manifest +def _process_sources_for_report( + manifest: Manifest, current_project: str, report: ParsedReport +): + target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None + for source_name, table_name in report.sources: + target_source = manifest.resolve_source( + source_name, + table_name, + current_project, + report.package_name, + ) + if target_source is None or isinstance(target_source, Disabled): + invalid_source_fail_unless_test( + report, + source_name, + table_name, + disabled=(isinstance(target_source, Disabled)) + ) + continue + target_source_id = target_source.unique_id + report.depends_on.nodes.append(target_source_id) + manifest.update_report(report) + + def _process_sources_for_node( - manifest: Manifest, current_project: str, node: NonSourceNode + manifest: Manifest, current_project: str, node: ManifestNode ): target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None for source_name, table_name in node.sources: @@ -636,6 +704,8 @@ def process_sources(manifest: Manifest, current_project: str): continue assert not isinstance(node, ParsedSourceDefinition) _process_sources_for_node(manifest, current_project, node) + for report in manifest.reports.values(): + _process_sources_for_report(manifest, current_project, report) return manifest @@ -652,7 +722,7 @@ def process_macro( def process_node( - config: RuntimeConfig, manifest: Manifest, node: NonSourceNode + config: RuntimeConfig, manifest: Manifest, node: ManifestNode ): _process_sources_for_node( diff --git a/core/dbt/parser/results.py b/core/dbt/parser/results.py index 33d2cdc30d1..44ff65f9798 100644 --- a/core/dbt/parser/results.py +++ b/core/dbt/parser/results.py @@ -15,6 +15,7 @@ ParsedMacroPatch, ParsedModelNode, ParsedNodePatch, + ParsedReport, ParsedRPCNode, ParsedSeedNode, ParsedSchemaTestNode, @@ -69,6 +70,7 @@ class ParseResult(JsonSchemaMixin, Writable, Replaceable): sources: MutableMapping[str, UnpatchedSourceDefinition] = dict_field() docs: MutableMapping[str, ParsedDocumentation] = dict_field() macros: MutableMapping[str, ParsedMacro] = dict_field() + reports: MutableMapping[str, ParsedReport] = dict_field() macro_patches: MutableMapping[MacroKey, ParsedMacroPatch] = dict_field() patches: MutableMapping[str, ParsedNodePatch] = dict_field() source_patches: MutableMapping[SourceKey, SourcePatch] = dict_field() @@ -101,6 +103,11 @@ def add_node(self, source_file: SourceFile, node: ManifestNodes): self.add_node_nofile(node) self.get_file(source_file).nodes.append(node.unique_id) + def add_report(self, source_file: SourceFile, report: ParsedReport): + _check_duplicates(report, self.reports) + self.reports[report.unique_id] = report + self.get_file(source_file).reports.append(report.unique_id) + def add_disabled_nofile(self, node: CompileResultNode): if node.unique_id in self.disabled: self.disabled[node.unique_id].append(node) @@ -262,6 +269,12 @@ def sanitized_update( continue self._process_node(node_id, source_file, old_file, old_result) + for report_id in old_file.reports: + report = _expect_value( + report_id, old_result.reports, old_file, "reports" + ) + self.add_report(source_file, report) + patched = False for name in old_file.patches: patch = _expect_value( diff --git a/core/dbt/parser/schema_test_builders.py b/core/dbt/parser/schema_test_builders.py index 24f649a3eb7..81668845b57 100644 --- a/core/dbt/parser/schema_test_builders.py +++ b/core/dbt/parser/schema_test_builders.py @@ -9,7 +9,11 @@ from dbt.clients.jinja import get_rendered, SCHEMA_TEST_KWARGS_NAME from dbt.contracts.graph.parsed import UnpatchedSourceDefinition from dbt.contracts.graph.unparsed import ( - UnparsedNodeUpdate, UnparsedMacroUpdate, UnparsedAnalysisUpdate, TestDef, + TestDef, + UnparsedAnalysisUpdate, + UnparsedMacroUpdate, + UnparsedNodeUpdate, + UnparsedReport, ) from dbt.exceptions import raise_compiler_error from dbt.parser.search import FileBlock @@ -78,6 +82,7 @@ def from_file_block(cls, src: FileBlock, data: Dict[str, Any]): UnparsedMacroUpdate, UnparsedAnalysisUpdate, UnpatchedSourceDefinition, + UnparsedReport, ) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index ddffea431ca..1603d63f5a7 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -18,6 +18,7 @@ ) from dbt.context.configured import generate_schema_yml from dbt.context.target import generate_target_context +from dbt.context.providers import generate_parse_report from dbt.contracts.files import FileHash from dbt.contracts.graph.manifest import SourceFile from dbt.contracts.graph.model_config import SourceConfig @@ -28,11 +29,20 @@ ParsedSchemaTestNode, ParsedMacroPatch, UnpatchedSourceDefinition, + ParsedReport, ) from dbt.contracts.graph.unparsed import ( - UnparsedSourceDefinition, UnparsedNodeUpdate, UnparsedColumn, - UnparsedMacroUpdate, UnparsedAnalysisUpdate, SourcePatch, - HasDocs, HasColumnDocs, HasColumnTests, FreshnessThreshold, + FreshnessThreshold, + HasColumnDocs, + HasColumnTests, + HasDocs, + SourcePatch, + UnparsedAnalysisUpdate, + UnparsedColumn, + UnparsedMacroUpdate, + UnparsedNodeUpdate, + UnparsedReport, + UnparsedSourceDefinition, ) from dbt.exceptions import ( validator_error_message, JSONValidationException, @@ -511,6 +521,11 @@ def parse_tests(self, block: TestBlock) -> None: for test in block.tests: self.parse_test(block, test, None) + def parse_reports(self, block: YamlBlock) -> None: + parser = ReportParser(self, block) + for node in parser.parse(): + self.results.add_report(block.file, node) + def parse_file(self, block: FileBlock) -> None: dct = self._yaml_from_file(block.file) # mark the file as seen, even if there are no macros in it @@ -541,6 +556,7 @@ def parse_file(self, block: FileBlock) -> None: parser = TestablePatchParser(self, yaml_block, plural) for test_block in parser.parse(): self.parse_tests(test_block) + self.parse_reports(yaml_block) Parsed = TypeVar( @@ -557,7 +573,7 @@ def parse_file(self, block: FileBlock) -> None: ) -class YamlDocsReader(metaclass=ABCMeta): +class YamlReader(metaclass=ABCMeta): def __init__( self, schema_parser: SchemaParser, yaml: YamlBlock, key: str ) -> None: @@ -599,6 +615,8 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: ) raise CompilationException(msg) + +class YamlDocsReader(YamlReader): @abstractmethod def parse(self) -> List[TestBlock]: raise NotImplementedError('parse is abstract') @@ -763,3 +781,57 @@ def parse_patch( docs=block.target.docs, ) self.results.add_macro_patch(self.yaml.file, result) + + +class ReportParser(YamlReader): + def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock): + super().__init__(schema_parser, yaml, NodeType.Report.pluralize()) + self.schema_parser = schema_parser + self.yaml = yaml + + def parse_report(self, unparsed: UnparsedReport) -> ParsedReport: + package_name = self.project.project_name + unique_id = f'{NodeType.Report}.{package_name}.{unparsed.name}' + path = self.yaml.path.relative_path + + fqn = self.schema_parser.get_fqn_prefix(path) + fqn.append(unparsed.name) + + parsed = ParsedReport( + package_name=package_name, + root_path=self.project.project_root, + path=path, + original_file_path=self.yaml.path.original_file_path, + unique_id=unique_id, + fqn=fqn, + name=unparsed.name, + type=unparsed.type, + url=unparsed.url, + description=unparsed.description, + owner=unparsed.owner, + maturity=unparsed.maturity, + ) + ctx = generate_parse_report( + parsed, + self.root_project, + self.schema_parser.macro_manifest, + package_name, + ) + depends_on_jinja = '\n'.join( + '{{ ' + line + '}}' for line in unparsed.depends_on + ) + get_rendered( + depends_on_jinja, ctx, parsed, capture_macros=True + ) + # parsed now has a populated refs/sources + return parsed + + def parse(self) -> Iterable[ParsedReport]: + for data in self.get_key_dicts(): + try: + unparsed = UnparsedReport.from_dict(data) + except (ValidationError, JSONValidationException) as exc: + msg = error_context(self.yaml.path, self.key, data, exc) + raise CompilationException(msg) from exc + parsed = self.parse_report(unparsed) + yield parsed diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 229927751dd..9e563fd0223 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -1,6 +1,10 @@ import json from typing import Type +from dbt.contracts.graph.parsed import ( + ParsedReport, + ParsedSourceDefinition, +) from dbt.graph import ( parse_difference, ResourceTypeSelector, @@ -20,6 +24,7 @@ class ListTask(GraphRunnableTask): NodeType.Seed, NodeType.Test, NodeType.Source, + NodeType.Report, )) ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset(( NodeType.Analysis, @@ -71,6 +76,8 @@ def _iterate_selected_nodes(self): yield self.manifest.nodes[node] elif node in self.manifest.sources: yield self.manifest.sources[node] + elif node in self.manifest.reports: + yield self.manifest.reports[node] else: raise RuntimeException( f'Got an unexpected result from node selection: "{node}"' @@ -79,18 +86,25 @@ def _iterate_selected_nodes(self): def generate_selectors(self): for node in self._iterate_selected_nodes(): - selector = '.'.join(node.fqn) if node.resource_type == NodeType.Source: - yield 'source:{}'.format(selector) + assert isinstance(node, ParsedSourceDefinition) + # sources are searched for by pkg.source_name.table_name + source_selector = '.'.join([ + node.package_name, node.source_name, node.name + ]) + yield f'source:{source_selector}' + elif node.resource_type == NodeType.Report: + assert isinstance(node, ParsedReport) + # reports are searched for by pkg.report_name + report_selector = '.'.join([node.package_name, node.name]) + yield f'report:{report_selector}' else: - yield selector + # everything else is from `fqn` + yield '.'.join(node.fqn) def generate_names(self): for node in self._iterate_selected_nodes(): - if node.resource_type == NodeType.Source: - yield '{0.source_name}.{0.name}'.format(node) - else: - yield node.name + yield node.search_name def generate_json(self): for node in self._iterate_selected_nodes(): diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index f37c70c2fa9..44fec0ac6f4 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -115,10 +115,14 @@ def __init__(self, graph, manifest, previous_state): ) def expand_selection(self, selected: Set[UniqueId]) -> Set[UniqueId]: - selected_tests = { - n for n in self.graph.select_successors(selected) - if self.manifest.nodes[n].resource_type == NodeType.Test - } + # reports can't have tests, so this is relatively easy + selected_tests = set() + for unique_id in self.graph.select_successors(selected): + if unique_id in self.manifest.nodes: + node = self.manifest.nodes[unique_id] + if node.resource_type == NodeType.Test: + selected_tests.add(unique_id) + return selected | selected_tests diff --git a/core/dbt/version.py b/core/dbt/version.py index 17349406e14..5bd39feda72 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -96,5 +96,5 @@ def _get_dbt_plugins_info(): yield plugin_name, mod.version -__version__ = '0.18.0' +__version__ = '0.18.1a1' installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 7d18cd70a79..82571a505ed 100644 --- a/core/setup.py +++ b/core/setup.py @@ -24,7 +24,7 @@ def read(fname): package_name = "dbt-core" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively""" diff --git a/plugins/bigquery/dbt/adapters/bigquery/__version__.py b/plugins/bigquery/dbt/adapters/bigquery/__version__.py index ec1bb8e9f8d..3e662f03665 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/__version__.py +++ b/plugins/bigquery/dbt/adapters/bigquery/__version__.py @@ -1 +1 @@ -version = '0.18.0' +version = '0.18.1a1' diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index 4324d8982ab..a8ef1ccb954 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -20,7 +20,7 @@ package_name = "dbt-bigquery" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """The bigquery adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/postgres/dbt/adapters/postgres/__version__.py b/plugins/postgres/dbt/adapters/postgres/__version__.py index ec1bb8e9f8d..3e662f03665 100644 --- a/plugins/postgres/dbt/adapters/postgres/__version__.py +++ b/plugins/postgres/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = '0.18.0' +version = '0.18.1a1' diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index 833aa25a050..13e7f9b052a 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -41,7 +41,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """The postgres adpter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/redshift/dbt/adapters/redshift/__version__.py b/plugins/redshift/dbt/adapters/redshift/__version__.py index ec1bb8e9f8d..3e662f03665 100644 --- a/plugins/redshift/dbt/adapters/redshift/__version__.py +++ b/plugins/redshift/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = '0.18.0' +version = '0.18.1a1' diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index 24d1ecd5169..ac46ba52986 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -20,7 +20,7 @@ package_name = "dbt-redshift" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """The redshift adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/snowflake/dbt/adapters/snowflake/__version__.py b/plugins/snowflake/dbt/adapters/snowflake/__version__.py index ec1bb8e9f8d..3e662f03665 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/__version__.py +++ b/plugins/snowflake/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = '0.18.0' +version = '0.18.1a1' diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py index 63fc015afb3..e7083a1e46e 100644 --- a/plugins/snowflake/setup.py +++ b/plugins/snowflake/setup.py @@ -20,7 +20,7 @@ package_name = "dbt-snowflake" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """The snowflake adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/setup.py b/setup.py index bbf073e5155..9070952db3c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ package_name = "dbt" -package_version = "0.18.0" +package_version = "0.18.1a1" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" diff --git a/test/integration/007_graph_selection_tests/models/base_users.sql b/test/integration/007_graph_selection_tests/models/base_users.sql index 858eb5ba343..5e048806d9a 100644 --- a/test/integration/007_graph_selection_tests/models/base_users.sql +++ b/test/integration/007_graph_selection_tests/models/base_users.sql @@ -6,4 +6,4 @@ ) }} -select * from {{ this.schema }}.seed +select * from {{ source('raw', 'seed') }} diff --git a/test/integration/007_graph_selection_tests/models/schema.yml b/test/integration/007_graph_selection_tests/models/schema.yml index 9f4f699a926..b790dca4e6f 100644 --- a/test/integration/007_graph_selection_tests/models/schema.yml +++ b/test/integration/007_graph_selection_tests/models/schema.yml @@ -1,18 +1,39 @@ version: 2 models: -- name: emails - columns: - - name: email - tests: - - not_null: - severity: warn -- name: users - columns: - - name: id - tests: - - unique -- name: users_rollup - columns: - - name: gender - tests: - - unique + - name: emails + columns: + - name: email + tests: + - not_null: + severity: warn + - name: users + columns: + - name: id + tests: + - unique + - name: users_rollup + columns: + - name: gender + tests: + - unique + +sources: + - name: raw + schema: '{{ target.schema }}' + tables: + - name: seed + +reports: + - name: user_report + type: dashboard + depends_on: + - ref('users') + - ref('users_rollup') + owner: + email: nope@example.com + - name: seed_ml_report + type: ml + depends_on: + - source('raw', 'seed') + owner: + email: nope@example.com diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index a3624e7367f..5b38b1f24fd 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -355,3 +355,24 @@ def test__postgres__concat_exclude_concat(self): self.assertEqual(len(results), 1) assert results[0].node.name == 'unique_users_id' + @use_profile('postgres') + def test__postgres__report_parents(self): + self.run_sql_file("seed.sql") + results = self.run_dbt(['ls', '--select', '+report:seed_ml_report']) + assert len(results) == 2 + assert sorted(results) == ['report:test.seed_ml_report', 'source:test.raw.seed'] + + results = self.run_dbt(['ls', '--select', '1+report:user_report']) + assert len(results) == 3 + assert sorted(results) == ['report:test.user_report', 'test.users', 'test.users_rollup'] + + self.run_dbt(['run', '-m', '+report:user_report']) + # users, users_rollup + assert len(results) == 3 + + created_models = self.get_models_in_schema() + self.assertIn('users_rollup', created_models) + self.assertIn('users', created_models) + self.assertNotIn('emails_alt', created_models) + self.assertNotIn('subdir', created_models) + self.assertNotIn('nested_users', created_models) diff --git a/test/integration/029_docs_generate_tests/models/schema.yml b/test/integration/029_docs_generate_tests/models/schema.yml index 0331e0fc038..9b5d87b2163 100644 --- a/test/integration/029_docs_generate_tests/models/schema.yml +++ b/test/integration/029_docs_generate_tests/models/schema.yml @@ -37,3 +37,41 @@ models: description: The user's IP address - name: updated_at description: The last time this user's email was updated + + +sources: + - name: my_source + description: "My source" + loader: a_loader + schema: "{{ var('test_schema') }}" + tables: + - name: my_table + description: "My table" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "An ID field" + + +reports: + - name: simple_report + type: dashboard + depends_on: + - ref('model') + - source('my_source', 'my_table') + owner: + email: something@example.com + - name: notebook_report + type: notebook + depends_on: + - ref('model') + - ref('second_model') + owner: + email: something@example.com + name: Some name + description: > + A description of the complex report + maturity: medium + url: http://example.com/notebook/1 diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index a8e274467ff..d777704eb8d 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -95,6 +95,12 @@ def test_postgres_include_schema(self): class TestDocsGenerate(DBTIntegrationTest): setup_alternate_db = True + def adapter_case(self, value): + if self.adapter_type == 'snowflake': + return value.upper() + else: + return value.lower() + def setUp(self): super().setUp() self.maxDiff = None @@ -196,7 +202,7 @@ def _redshift_stats(self): "diststyle": { "id": "diststyle", "label": "Dist Style", - "value": AnyStringWith(None), + "value": AnyStringWith('AUTO'), "description": "Distribution style or distribution key column, if key distribution is defined.", "include": True }, @@ -214,6 +220,13 @@ def _redshift_stats(self): "description": "Approximate size of the table, calculated from a count of 1MB blocks", "include": True }, + 'sortkey1': { + 'id': 'sortkey1', + 'label': 'Sort Key 1', + 'value': AnyStringWith('AUTO'), + 'description': 'First column in the sort key.', + 'include': True, + }, "pct_used": { "id": "pct_used", "label": "Disk Utilization", @@ -404,7 +417,21 @@ def _expected_catalog(self, id_type, text_type, time_type, view_type, 'columns': expected_cols, }, }, - 'sources': {} + 'sources': { + 'source.test.my_source.my_table': { + 'unique_id': 'source.test.my_source.my_table', + 'metadata': { + 'schema': my_schema_name, + 'database': self.default_database, + 'name': case('seed'), + 'type': table_type, + 'comment': None, + 'owner': role, + }, + 'stats': seed_stats, + 'columns': expected_cols, + }, + }, } def expected_postgres_catalog(self): @@ -1098,7 +1125,6 @@ def expected_seeded_manifest(self, model_database=None): 'injected_sql': ANY, 'checksum': self._checksum_file(second_model_sql_path), }, - 'seed.test.seed': { 'build_path': None, 'compiled': True, @@ -1366,23 +1392,128 @@ def expected_seeded_manifest(self, model_database=None): 'checksum': {'name': 'none', 'checksum': ''}, }, }, - 'sources': {}, + 'sources': { + 'source.test.my_source.my_table': { + 'columns': { + 'id': { + 'description': 'An ID field', + 'name': 'id', + 'data_type': None, + 'meta': {}, + 'quote': None, + 'tags': [], + } + }, + 'config': { + 'enabled': True, + }, + 'quoting': { + 'database': None, + 'schema': None, + 'identifier': True, + 'column': None, + }, + 'database': self.default_database, + 'description': 'My table', + 'external': None, + 'freshness': {'error_after': None, 'warn_after': None, 'filter': None}, + 'identifier': 'seed', + 'loaded_at_field': None, + 'loader': 'a_loader', + 'meta': {}, + 'name': 'my_table', + 'original_file_path': self.dir('models/schema.yml'), + 'package_name': 'test', + 'path': self.dir('models/schema.yml'), + 'patch_path': None, + 'resource_type': 'source', + 'root_path': self.test_root_realpath, + 'schema': my_schema_name, + 'source_description': 'My source', + 'source_name': 'my_source', + 'source_meta': {}, + 'tags': [], + 'unique_id': 'source.test.my_source.my_table', + 'fqn': ['test', 'my_source', 'my_table'], + }, + }, + 'reports': { + 'report.test.notebook_report': { + 'depends_on': { + 'macros': [], + 'nodes': ['model.test.model', 'model.test.second_model'] + }, + 'description': 'A description of the complex report', + 'fqn': ['test', 'notebook_report'], + 'maturity': 'medium', + 'name': 'notebook_report', + 'original_file_path': self.dir('models/schema.yml'), + 'owner': { + 'email': 'something@example.com', + 'name': 'Some name' + }, + 'package_name': 'test', + 'path': 'schema.yml', + 'refs': [['model'], ['second_model']], + 'resource_type': 'report', + 'root_path': self.test_root_realpath, + 'sources': [], + 'type': 'notebook', + 'unique_id': 'report.test.notebook_report', + 'url': 'http://example.com/notebook/1' + }, + 'report.test.simple_report': { + 'depends_on': { + 'macros': [], + 'nodes': [ + 'source.test.my_source.my_table', + 'model.test.model' + ], + }, + 'description': None, + 'fqn': ['test', 'simple_report'], + 'name': 'simple_report', + 'original_file_path': self.dir('models/schema.yml'), + 'owner': { + 'email': 'something@example.com', + 'name': None, + }, + 'package_name': 'test', + 'path': 'schema.yml', + 'refs': [['model']], + 'resource_type': 'report', + 'root_path': self.test_root_realpath, + 'sources': [['my_source', 'my_table']], + 'type': 'dashboard', + 'unique_id': 'report.test.simple_report', + 'url': None, + 'maturity': None, + } + }, 'parent_map': { 'model.test.model': ['seed.test.seed'], 'model.test.second_model': ['seed.test.seed'], + 'report.test.notebook_report': ['model.test.model', 'model.test.second_model'], + 'report.test.simple_report': ['model.test.model', 'source.test.my_source.my_table'], 'seed.test.seed': [], + 'source.test.my_source.my_table': [], 'test.test.not_null_model_id': ['model.test.model'], 'test.test.test_nothing_model_': ['model.test.model'], 'test.test.unique_model_id': ['model.test.model'], }, 'child_map': { 'model.test.model': [ + 'report.test.notebook_report', + 'report.test.simple_report', 'test.test.not_null_model_id', 'test.test.test_nothing_model_', 'test.test.unique_model_id', ], - 'model.test.second_model': [], + 'model.test.second_model': ['report.test.notebook_report'], + 'report.test.notebook_report': [], + 'report.test.simple_report': [], 'seed.test.seed': ['model.test.model', 'model.test.second_model'], + 'source.test.my_source.my_table': ['report.test.simple_report'], 'test.test.not_null_model_id': [], 'test.test.test_nothing_model_': [], 'test.test.unique_model_id': [], @@ -1740,6 +1871,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'fqn': ['test', 'my_source', 'my_table'], }, }, + 'reports': {}, 'docs': { 'dbt.__overview__': ANY, 'test.column_info': { @@ -2299,6 +2431,7 @@ def expected_bigquery_complex_manifest(self): }, }, 'sources': {}, + 'reports': {}, 'child_map': { 'model.test.clustered': [], 'model.test.multi_clustered': [], @@ -2543,6 +2676,7 @@ def expected_redshift_incremental_view_manifest(self): }, }, 'sources': {}, + 'reports': {}, 'parent_map': { 'model.test.model': ['seed.test.seed'], 'seed.test.seed': [] @@ -2572,7 +2706,7 @@ def verify_manifest(self, expected_manifest): manifest_keys = frozenset({ 'nodes', 'sources', 'macros', 'parent_map', 'child_map', 'generated_at', - 'docs', 'metadata', 'docs', 'disabled' + 'docs', 'metadata', 'docs', 'disabled', 'reports' }) self.assertEqual(frozenset(manifest), manifest_keys) diff --git a/test/integration/048_rpc_test/error_models/model.sql b/test/integration/048_rpc_test/error_models/model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/048_rpc_test/error_models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/048_rpc_test/error_models/schema.yml b/test/integration/048_rpc_test/error_models/schema.yml deleted file mode 100644 index 69cf1f304a6..00000000000 --- a/test/integration/048_rpc_test/error_models/schema.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - freshness: - warn_after: {count: 10, period: hour} - error_after: {count: 1, period: day} - schema: invalid - tables: - - name: test_table - identifier: source - loaded_at_field: updated_at diff --git a/test/integration/048_rpc_test/malformed_models/schema.yml b/test/integration/048_rpc_test/malformed_models/schema.yml deleted file mode 100644 index 6962204bc7f..00000000000 --- a/test/integration/048_rpc_test/malformed_models/schema.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -sources: - - name: test_source - loader: custom - schema: "{{ var('test_run_schema') }}" - tables: - - name: test_table - identifier: source - tests: - - relationships: - # this is invalid - - column_name: favorite_color - - to: ref('descendant_model') - - field: favorite_color diff --git a/test/integration/048_rpc_test/models/descendant_model.sql b/test/integration/048_rpc_test/models/descendant_model.sql deleted file mode 100644 index 55bbcba67b4..00000000000 --- a/test/integration/048_rpc_test/models/descendant_model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ source('test_source', 'test_table') }} diff --git a/test/integration/062_defer_state_test/models/reports.yml b/test/integration/062_defer_state_test/models/reports.yml new file mode 100644 index 00000000000..19d52057d77 --- /dev/null +++ b/test/integration/062_defer_state_test/models/reports.yml @@ -0,0 +1,8 @@ +version: 2 +reports: + - name: my_report + type: application + depends_on: + - ref('view_model') + owner: + email: test@example.com diff --git a/test/integration/062_defer_state_test/test_modified_state.py b/test/integration/062_defer_state_test/test_modified_state.py index 535a9d11eec..4c64c387cb5 100644 --- a/test/integration/062_defer_state_test/test_modified_state.py +++ b/test/integration/062_defer_state_test/test_modified_state.py @@ -71,8 +71,8 @@ def test_postgres_changed_seed_contents_state(self): assert results[0] == 'test.seed' results = self.run_dbt(['ls', '--select', 'state:modified+', '--state', './state']) - assert len(results) == 6 - assert set(results) == {'test.seed', 'test.table_model', 'test.view_model', 'test.ephemeral_model', 'test.schema_test.not_null_view_model_id', 'test.schema_test.unique_view_model_id'} + assert len(results) == 7 + assert set(results) == {'test.seed', 'test.table_model', 'test.view_model', 'test.ephemeral_model', 'test.schema_test.not_null_view_model_id', 'test.schema_test.unique_view_model_id', 'report:test.my_report'} shutil.rmtree('./state') self.copy_state() @@ -183,3 +183,12 @@ def test_postgres_changed_macro_contents(self): results, stdout = self.run_dbt_and_capture(['run', '--models', 'state:modified', '--state', './state'], strict=False) assert len(results) == 0 assert 'detected a change in macros' in stdout + + @use_profile('postgres') + def test_postgres_changed_report(self): + with open('models/reports.yml', 'a') as fp: + fp.write(' name: John Doe\n') + + results, stdout = self.run_dbt_and_capture(['run', '--models', '+state:modified', '--state', './state']) + assert len(results) == 1 + assert results[0].node.name == 'view_model' diff --git a/test/integration/048_rpc_test/data/expected_multi_source.csv b/test/integration/100_rpc_test/data/expected_multi_source.csv similarity index 100% rename from test/integration/048_rpc_test/data/expected_multi_source.csv rename to test/integration/100_rpc_test/data/expected_multi_source.csv diff --git a/test/integration/048_rpc_test/data/other_source_table.csv b/test/integration/100_rpc_test/data/other_source_table.csv similarity index 100% rename from test/integration/048_rpc_test/data/other_source_table.csv rename to test/integration/100_rpc_test/data/other_source_table.csv diff --git a/test/integration/048_rpc_test/data/other_table.csv b/test/integration/100_rpc_test/data/other_table.csv similarity index 100% rename from test/integration/048_rpc_test/data/other_table.csv rename to test/integration/100_rpc_test/data/other_table.csv diff --git a/test/integration/048_rpc_test/data/source.csv b/test/integration/100_rpc_test/data/source.csv similarity index 100% rename from test/integration/048_rpc_test/data/source.csv rename to test/integration/100_rpc_test/data/source.csv diff --git a/test/integration/048_rpc_test/deps_models/main.sql b/test/integration/100_rpc_test/deps_models/main.sql similarity index 100% rename from test/integration/048_rpc_test/deps_models/main.sql rename to test/integration/100_rpc_test/deps_models/main.sql diff --git a/test/integration/048_rpc_test/macros/macro.sql b/test/integration/100_rpc_test/macros/macro.sql similarity index 100% rename from test/integration/048_rpc_test/macros/macro.sql rename to test/integration/100_rpc_test/macros/macro.sql diff --git a/test/integration/048_rpc_test/malformed_models/descendant_model.sql b/test/integration/100_rpc_test/models/descendant_model.sql similarity index 100% rename from test/integration/048_rpc_test/malformed_models/descendant_model.sql rename to test/integration/100_rpc_test/models/descendant_model.sql diff --git a/test/integration/048_rpc_test/models/ephemeral_model.sql b/test/integration/100_rpc_test/models/ephemeral_model.sql similarity index 100% rename from test/integration/048_rpc_test/models/ephemeral_model.sql rename to test/integration/100_rpc_test/models/ephemeral_model.sql diff --git a/test/integration/048_rpc_test/models/multi_source_model.sql b/test/integration/100_rpc_test/models/multi_source_model.sql similarity index 100% rename from test/integration/048_rpc_test/models/multi_source_model.sql rename to test/integration/100_rpc_test/models/multi_source_model.sql diff --git a/test/integration/048_rpc_test/models/nonsource_descendant.sql b/test/integration/100_rpc_test/models/nonsource_descendant.sql similarity index 100% rename from test/integration/048_rpc_test/models/nonsource_descendant.sql rename to test/integration/100_rpc_test/models/nonsource_descendant.sql diff --git a/test/integration/048_rpc_test/models/schema.yml b/test/integration/100_rpc_test/models/schema.yml similarity index 100% rename from test/integration/048_rpc_test/models/schema.yml rename to test/integration/100_rpc_test/models/schema.yml diff --git a/test/integration/048_rpc_test/seed.sql b/test/integration/100_rpc_test/seed.sql similarity index 100% rename from test/integration/048_rpc_test/seed.sql rename to test/integration/100_rpc_test/seed.sql diff --git a/test/integration/048_rpc_test/sql/bigquery.sql b/test/integration/100_rpc_test/sql/bigquery.sql similarity index 100% rename from test/integration/048_rpc_test/sql/bigquery.sql rename to test/integration/100_rpc_test/sql/bigquery.sql diff --git a/test/integration/048_rpc_test/sql/redshift.sql b/test/integration/100_rpc_test/sql/redshift.sql similarity index 100% rename from test/integration/048_rpc_test/sql/redshift.sql rename to test/integration/100_rpc_test/sql/redshift.sql diff --git a/test/integration/048_rpc_test/sql/snowflake.sql b/test/integration/100_rpc_test/sql/snowflake.sql similarity index 100% rename from test/integration/048_rpc_test/sql/snowflake.sql rename to test/integration/100_rpc_test/sql/snowflake.sql diff --git a/test/integration/048_rpc_test/test_execute_fetch_and_serialize.py b/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py similarity index 97% rename from test/integration/048_rpc_test/test_execute_fetch_and_serialize.py rename to test/integration/100_rpc_test/test_execute_fetch_and_serialize.py index 6ba315afa02..d5fa2ba76fe 100644 --- a/test/integration/048_rpc_test/test_execute_fetch_and_serialize.py +++ b/test/integration/100_rpc_test/test_execute_fetch_and_serialize.py @@ -7,7 +7,7 @@ class TestRpcExecuteReturnsResults(DBTIntegrationTest): @property def schema(self): - return "rpc_test_048" + return "rpc_test_100" @property def models(self): diff --git a/test/integration/048_rpc_test/test_rpc.py b/test/integration/100_rpc_test/test_rpc.py similarity index 99% rename from test/integration/048_rpc_test/test_rpc.py rename to test/integration/100_rpc_test/test_rpc.py index 912876624df..9b722f1837f 100644 --- a/test/integration/048_rpc_test/test_rpc.py +++ b/test/integration/100_rpc_test/test_rpc.py @@ -131,7 +131,7 @@ def tearDown(self): @property def schema(self): - return "rpc_048" + return "rpc_100" @property def models(self): diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index 367462a08a3..c25515eecc6 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -156,6 +156,7 @@ def test__prepend_ctes__already_has_cte(self): generated_at=datetime(2018, 2, 14, 9, 15, 13), disabled=[], files={}, + reports={}, ) compiler = dbt.compilation.Compiler(self.config) @@ -240,6 +241,7 @@ def test__prepend_ctes__no_ctes(self): generated_at='2018-02-14T09:15:13Z', disabled=[], files={}, + reports={}, ) compiler = dbt.compilation.Compiler(self.config) @@ -333,6 +335,7 @@ def test__prepend_ctes(self): generated_at='2018-02-14T09:15:13Z', disabled=[], files={}, + reports={}, ) compiler = dbt.compilation.Compiler(self.config) @@ -437,6 +440,7 @@ def test__prepend_ctes__cte_not_compiled(self): generated_at='2018-02-14T09:15:13Z', disabled=[], files={}, + reports={}, ) compiler = dbt.compilation.Compiler(self.config) @@ -542,6 +546,7 @@ def test__prepend_ctes__multiple_levels(self): generated_at='2018-02-14T09:15:13Z', disabled=[], files={}, + reports={}, ) compiler = dbt.compilation.Compiler(self.config) diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 48db6e053c9..0eeea70085c 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -24,15 +24,24 @@ IntermediateSnapshotNode, ParsedNodePatch, ParsedMacro, + ParsedReport, ParsedSeedNode, Docs, MacroDependsOn, ParsedSourceDefinition, ParsedDocumentation, ParsedHookNode, + ReportOwner, TestMetadata, ) -from dbt.contracts.graph.unparsed import Quoting, Time, TimePeriod, FreshnessThreshold +from dbt.contracts.graph.unparsed import ( + ExposureType, + FreshnessThreshold, + MaturityType, + Quoting, + Time, + TimePeriod, +) from dbt import flags from hologram import ValidationError @@ -579,7 +588,6 @@ def test_compare_changed_seed(func, basic_parsed_seed_object): assert not node.same_contents(compare) - @pytest.fixture def basic_parsed_model_patch_dict(): return { @@ -1863,3 +1871,143 @@ def test_compare_changed_source_definition(func, basic_parsed_source_definition_ node, compare = func(basic_parsed_source_definition_object) assert not node.same_contents(compare) + +@pytest.fixture +def minimal_parsed_report_dict(): + return { + 'name': 'my_report', + 'type': 'notebook', + 'owner': { + 'email': 'test@example.com', + }, + 'fqn': ['test', 'reports', 'my_report'], + 'unique_id': 'report.test.my_report', + 'package_name': 'test', + 'path': 'models/something.yml', + 'root_path': '/usr/src/app', + 'original_file_path': 'models/something.yml', + } + + +@pytest.fixture +def basic_parsed_report_dict(): + return { + 'name': 'my_report', + 'type': 'notebook', + 'owner': { + 'email': 'test@example.com', + }, + 'resource_type': 'report', + 'depends_on': { + 'nodes': [], + 'macros': [], + }, + 'refs': [], + 'sources': [], + 'fqn': ['test', 'reports', 'my_report'], + 'unique_id': 'report.test.my_report', + 'package_name': 'test', + 'path': 'models/something.yml', + 'root_path': '/usr/src/app', + 'original_file_path': 'models/something.yml', + } + + +@pytest.fixture +def basic_parsed_report_object(): + return ParsedReport( + name='my_report', + type=ExposureType.Notebook, + fqn=['test', 'reports', 'my_report'], + unique_id='report.test.my_report', + package_name='test', + path='models/something.yml', + root_path='/usr/src/app', + original_file_path='models/something.yml', + owner=ReportOwner(email='test@example.com'), + ) + + +@pytest.fixture +def complex_parsed_report_dict(): + return { + 'name': 'my_report', + 'type': 'analysis', + 'owner': { + 'email': 'test@example.com', + 'name': 'A Name', + }, + 'resource_type': 'report', + 'maturity': 'low', + 'url': 'https://example.com/analyses/1', + 'description': 'my description', + 'depends_on': { + 'nodes': ['models.test.my_model'], + 'macros': [], + }, + 'refs': [], + 'sources': [], + 'fqn': ['test', 'reports', 'my_report'], + 'unique_id': 'report.test.my_report', + 'package_name': 'test', + 'path': 'models/something.yml', + 'root_path': '/usr/src/app', + 'original_file_path': 'models/something.yml', + } + + +@pytest.fixture +def complex_parsed_report_object(): + return ParsedReport( + name='my_report', + type=ExposureType.Analysis, + owner=ReportOwner(email='test@example.com', name='A Name'), + maturity=MaturityType.Low, + url='https://example.com/analyses/1', + description='my description', + depends_on=DependsOn(nodes=['models.test.my_model']), + fqn=['test', 'reports', 'my_report'], + unique_id='report.test.my_report', + package_name='test', + path='models/something.yml', + root_path='/usr/src/app', + original_file_path='models/something.yml', + ) + + +def test_basic_parsed_report(minimal_parsed_report_dict, basic_parsed_report_dict, basic_parsed_report_object): + assert_symmetric(basic_parsed_report_object, basic_parsed_report_dict, ParsedReport) + assert_from_dict(basic_parsed_report_object, minimal_parsed_report_dict, ParsedReport) + pickle.loads(pickle.dumps(basic_parsed_report_object)) + + +def test_complex_parsed_report(complex_parsed_report_dict, complex_parsed_report_object): + assert_symmetric(complex_parsed_report_object, complex_parsed_report_dict, ParsedReport) + + +unchanged_parsed_reports = [ + lambda u: (u, u), +] + + +changed_parsed_reports = [ + lambda u: (u, u.replace(fqn=u.fqn[:-1]+['something', u.fqn[-1]])), + lambda u: (u, u.replace(type=ExposureType.ML)), + lambda u: (u, u.replace(owner=u.owner.replace(name='My Name'))), + lambda u: (u, u.replace(maturity=MaturityType.Medium)), + lambda u: (u, u.replace(url='https://example.com/dashboard/1')), + lambda u: (u, u.replace(description='My description')), + lambda u: (u, u.replace(depends_on=DependsOn(nodes=['model.test.blah']))), +] + + +@pytest.mark.parametrize('func', unchanged_parsed_reports) +def test_compare_unchanged_parsed_report(func, basic_parsed_report_object): + node, compare = func(basic_parsed_report_object) + assert node.same_contents(compare) + + +@pytest.mark.parametrize('func', changed_parsed_reports) +def test_compare_changed_report(func, basic_parsed_report_object): + node, compare = func(basic_parsed_report_object) + assert not node.same_contents(compare) diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index 1ddd8135822..65e7d75a8a5 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -1,3 +1,4 @@ +import copy import pickle from datetime import timedelta @@ -5,7 +6,8 @@ UnparsedNode, UnparsedRunHook, UnparsedMacro, Time, TimePeriod, FreshnessStatus, FreshnessThreshold, Quoting, UnparsedSourceDefinition, UnparsedSourceTableDefinition, UnparsedDocumentationFile, UnparsedColumn, - UnparsedNodeUpdate, Docs + UnparsedNodeUpdate, Docs, UnparsedReport, MaturityType, ReportOwner, + ExposureType ) from dbt.node_types import NodeType from .utils import ContractTestCase @@ -567,3 +569,78 @@ def test_bad_test_type(self): 'docs': {'show': True}, } self.assert_fails_validation(dct) + + +class TestUnparsedReport(ContractTestCase): + ContractType = UnparsedReport + + def get_ok_dict(self): + return { + 'name': 'my_report', + 'type': 'dashboard', + 'owner': { + 'email': 'name@example.com', + }, + 'maturity': 'medium', + 'url': 'https://example.com/dashboards/1', + 'description': 'A report', + 'depends_on': [ + 'ref("my_model")', + 'source("raw", "source_table")', + ] + } + + def test_ok(self): + report = self.ContractType( + name='my_report', + type=ExposureType.Dashboard, + owner=ReportOwner(email='name@example.com'), + maturity=MaturityType.Medium, + url='https://example.com/dashboards/1', + description='A report', + depends_on=['ref("my_model")', 'source("raw", "source_table")'], + ) + dct = self.get_ok_dict() + self.assert_symmetric(report, dct) + pickle.loads(pickle.dumps(report)) + + def test_ok_exposures(self): + for exposure_allowed in ('dashboard', 'notebook', 'analysis', 'ml', 'application'): + tst = self.get_ok_dict() + tst['type'] = exposure_allowed + assert self.ContractType.from_dict(tst).type == exposure_allowed + + def test_bad_exposure(self): + # bad exposure: None isn't allowed + for exposure_not_allowed in (None, 'not an exposure'): + tst = self.get_ok_dict() + tst['type'] = exposure_not_allowed + self.assert_fails_validation(tst) + + def test_no_exposure(self): + tst = self.get_ok_dict() + del tst['type'] + self.assert_fails_validation(tst) + + def test_ok_maturities(self): + for maturity_allowed in (None, 'low', 'medium', 'high'): + tst = self.get_ok_dict() + tst['maturity'] = maturity_allowed + assert self.ContractType.from_dict(tst).maturity == maturity_allowed + + tst = self.get_ok_dict() + del tst['maturity'] + assert self.ContractType.from_dict(tst).maturity is None + + def test_bad_maturity(self): + tst = self.get_ok_dict() + tst['maturity'] = 'invalid maturity' + self.assert_fails_validation(tst) + + def test_bad_owner_missing_things(self): + tst = self.get_ok_dict() + del tst['owner']['email'] + self.assert_fails_validation(tst) + + del tst['owner'] + self.assert_fails_validation(tst) diff --git a/test/unit/test_graph_selector_methods.py b/test/unit/test_graph_selector_methods.py index 127a3234fe2..007eaa3875f 100644 --- a/test/unit/test_graph_selector_methods.py +++ b/test/unit/test_graph_selector_methods.py @@ -10,6 +10,7 @@ DependsOn, NodeConfig, ParsedModelNode, + ParsedReport, ParsedSeedNode, ParsedSnapshotNode, ParsedDataTestNode, @@ -20,6 +21,7 @@ ColumnInfo, ) from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.unparsed import ExposureType, ReportOwner from dbt.contracts.state import PreviousState from dbt.node_types import NodeType from dbt.graph.selector_methods import ( @@ -33,6 +35,7 @@ TestNameSelectorMethod, TestTypeSelectorMethod, StateSelectorMethod, + ReportSelectorMethod, ) import dbt.exceptions import dbt.contracts.graph.parsed @@ -291,6 +294,30 @@ def make_data_test(pkg, name, sql, refs=None, sources=None, tags=None, path=None ) +def make_report(pkg, name, path=None, fqn_extras=None, owner=None): + if path is None: + path = 'schema.yml' + + if fqn_extras is None: + fqn_extras = [] + + if owner is None: + owner = ReportOwner(email='test@example.com') + + fqn = [pkg, 'reports'] + fqn_extras + [name] + return ParsedReport( + name=name, + type=ExposureType.Notebook, + fqn=fqn, + unique_id=f'report.{pkg}.{name}', + package_name=pkg, + path=path, + root_path='/usr/src/app', + original_file_path=path, + owner=owner, + ) + + @pytest.fixture def seed(): return make_seed( @@ -441,6 +468,7 @@ def manifest(seed, source, ephemeral_model, view_model, table_model, ext_source, macros={}, docs={}, files={}, + reports={}, generated_at=datetime.utcnow(), disabled=[], ) @@ -448,7 +476,7 @@ def manifest(seed, source, ephemeral_model, view_model, table_model, ext_source, def search_manifest_using_method(manifest, method, selection): - selected = method.search(set(manifest.nodes) | set(manifest.sources), selection) + selected = method.search(set(manifest.nodes) | set(manifest.sources) | set(manifest.reports), selection) results = {manifest.expect(uid).search_name for uid in selected} return results @@ -558,6 +586,16 @@ def test_select_test_type(manifest): assert search_manifest_using_method(manifest, method, 'data') == {'view_test_nothing'} +def test_select_report(manifest): + report = make_report('test', 'my_report') + manifest.reports[report.unique_id] = report + methods = MethodManager(manifest, None) + method = methods.get_method('report', []) + assert isinstance(method, ReportSelectorMethod) + assert search_manifest_using_method(manifest, method, 'my_report') == {'my_report'} + assert not search_manifest_using_method(manifest, method, 'not_my_report') + + @pytest.fixture def previous_state(manifest): writable = copy.deepcopy(manifest).writable_manifest() diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 9549df1f32c..49f856f8d52 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -215,13 +215,14 @@ def setUp(self): def test__no_nodes(self): manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) self.assertEqual( manifest.writable_manifest().to_dict(), { 'nodes': {}, 'sources': {}, 'macros': {}, + 'reports': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', @@ -236,7 +237,7 @@ def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) serialized = manifest.writable_manifest().to_dict() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') self.assertEqual(serialized['docs'], {}) @@ -302,7 +303,7 @@ def test__build_flat_graph(self): sources = copy.copy(self.sources) manifest = Manifest(nodes=nodes, sources=sources, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) manifest.build_flat_graph() flat_graph = manifest.flat_graph flat_nodes = flat_graph['nodes'] @@ -341,7 +342,7 @@ def test_no_nodes_with_metadata(self, mock_user): ) manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - metadata=metadata, files={}) + metadata=metadata, files={}, reports={}) self.assertEqual( manifest.writable_manifest().to_dict(), @@ -349,6 +350,7 @@ def test_no_nodes_with_metadata(self, mock_user): 'nodes': {}, 'sources': {}, 'macros': {}, + 'reports': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', @@ -366,7 +368,7 @@ def test_no_nodes_with_metadata(self, mock_user): def test_get_resource_fqns_empty(self): manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) self.assertEqual(manifest.get_resource_fqns(), {}) def test_get_resource_fqns(self): @@ -393,7 +395,7 @@ def test_get_resource_fqns(self): ) manifest = Manifest(nodes=nodes, sources=self.sources, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) expect = { 'models': frozenset([ ('snowplow', 'events'), @@ -575,13 +577,14 @@ def setUp(self): def test__no_nodes(self): manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) self.assertEqual( manifest.writable_manifest().to_dict(), { 'nodes': {}, 'macros': {}, 'sources': {}, + 'reports': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', @@ -596,7 +599,7 @@ def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) serialized = manifest.writable_manifest().to_dict() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') self.assertEqual(serialized['disabled'], []) @@ -660,7 +663,7 @@ def test__build_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, sources={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], - files={}) + files={}, reports={}) manifest.build_flat_graph() flat_graph = manifest.flat_graph flat_nodes = flat_graph['nodes'] @@ -707,7 +710,8 @@ def setUp(self): }, generated_at=datetime.utcnow(), disabled=[], - files={} + files={}, + reports={}, ) @@ -727,7 +731,8 @@ def make_manifest(nodes=[], sources=[], macros=[], docs=[]): }, generated_at=datetime.utcnow(), disabled=[], - files={} + files={}, + reports={}, ) diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 071f4b7eb67..51362ddeefc 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -834,7 +834,7 @@ def setUp(self): self.doc.unique_id: self.doc, } self.manifest = Manifest( - nodes=nodes, sources=sources, macros={}, docs=docs, disabled=[], files={}, generated_at=mock.MagicMock() + nodes=nodes, sources=sources, macros={}, docs=docs, disabled=[], files={}, reports={}, generated_at=mock.MagicMock() ) def test_process_docs(self):