Merge pull request #2752 from fishtown-analytics/feature/reports

Feature: reports

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.0
+current_version = 0.18.1a1
 parse = (?P<major>\d+)
  \.(?P<minor>\d+)
  \.(?P<patch>\d+)

CHANGELOG.md

@@ -5,6 +5,7 @@
 
 ### Features
 - Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https:/fishtown-analytics/dbt/pull/2680), [#2723](https:/fishtown-analytics/dbt/pull/2723))
+- Added "reports" ([#2730](https:/fishtown-analytics/dbt/issues/2730), [#2752](https:/fishtown-analytics/dbt/pull/2752))
 
 Contributors:
 - [@tpilewicz](https:/tpilewicz) ([#2723](https:/fishtown-analytics/dbt/pull/2723))

core/dbt/adapters/protocol.py

@@ -9,7 +9,7 @@
 
 from dbt.contracts.connection import Connection, AdapterRequiredConfig
 from dbt.contracts.graph.compiled import (
- CompiledNode, NonSourceNode, NonSourceCompiledNode
+ CompiledNode, ManifestNode, NonSourceCompiledNode
 )
 from dbt.contracts.graph.parsed import ParsedNode, ParsedSourceDefinition
 from dbt.contracts.graph.model_config import BaseConfig
@@ -55,7 +55,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph:
 
  def compile_node(
  self,
- node: NonSourceNode,
+ node: ManifestNode,
  manifest: Manifest,
  extra_context: Optional[Dict[str, Any]] = None,
  ) -> NonSourceCompiledNode:

core/dbt/compilation.py

@@ -12,12 +12,13 @@
 from dbt.context.providers import generate_runtime_model
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.compiled import (
- InjectedCTE,
- COMPILED_TYPES,
- NonSourceNode,
- NonSourceCompiledNode,
  CompiledDataTestNode,
  CompiledSchemaTestNode,
+ COMPILED_TYPES,
+ GraphMemberNode,
+ InjectedCTE,
+ ManifestNode,
+ NonSourceCompiledNode,
 )
 from dbt.contracts.graph.parsed import ParsedNode
 from dbt.exceptions import (
@@ -64,7 +65,7 @@ def print_compile_stats(stats):
  logger.info("Found {}".format(stat_line))
 
 
-def _node_enabled(node: NonSourceNode):
+def _node_enabled(node: ManifestNode):
  # Disabled models are already excluded from the manifest
  if node.resource_type == NodeType.Test and not node.config.enabled:
  return False
@@ -358,7 +359,7 @@ def _insert_ctes(
 
  def _compile_node(
  self,
- node: NonSourceNode,
+ node: ManifestNode,
  manifest: Manifest,
  extra_context: Optional[Dict[str, Any]] = None,
  ) -> NonSourceCompiledNode:
@@ -402,7 +403,7 @@ def write_graph_file(self, linker: Linker, manifest: Manifest):
  linker.write_graph(graph_path, manifest)
 
  def link_node(
- self, linker: Linker, node: NonSourceNode, manifest: Manifest
+ self, linker: Linker, node: GraphMemberNode, manifest: Manifest
  ):
  linker.add_node(node.unique_id)
 
@@ -425,6 +426,9 @@ def link_graph(self, linker: Linker, manifest: Manifest):
  linker.add_node(source.unique_id)
  for node in manifest.nodes.values():
  self.link_node(linker, node, manifest)
+ for report in manifest.reports.values():
+ self.link_node(linker, report, manifest)
+ # linker.add_node(report.unique_id)
 
  cycle = linker.find_cycles()
 
@@ -445,7 +449,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph:
 
  return Graph(linker.graph)
 
- def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode:
+ def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
  if not _is_writable(node):
  return node
  logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
@@ -467,7 +471,7 @@ def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode:
 
  def compile_node(
  self,
- node: NonSourceNode,
+ node: ManifestNode,
  manifest: Manifest,
  extra_context: Optional[Dict[str, Any]] = None,
  write: bool = True,

core/dbt/context/providers.py

@@ -21,10 +21,11 @@
 from dbt.contracts.graph.compiled import (
  CompiledResource,
  CompiledSeedNode,
- NonSourceNode,
+ ManifestNode,
 )
 from dbt.contracts.graph.parsed import (
  ParsedMacro,
+ ParsedReport,
  ParsedSeedNode,
  ParsedSourceDefinition,
 )
@@ -419,7 +420,7 @@ def resolve(
  return self.Relation.create_from(self.config, self.model)
 
 
-ResolveRef = Union[Disabled, NonSourceNode]
+ResolveRef = Union[Disabled, ManifestNode]
 
 
 class RuntimeRefResolver(BaseRefResolver):
@@ -444,7 +445,7 @@ def resolve(
  return self.create_relation(target_model, target_name)
 
  def create_relation(
- self, target_model: NonSourceNode, name: str
+ self, target_model: ManifestNode, name: str
  ) -> RelationProxy:
  if target_model.is_ephemeral_model:
  self.model.set_cte(target_model.unique_id, None)
@@ -456,7 +457,7 @@ def create_relation(
 
  def validate(
  self,
- resolved: NonSourceNode,
+ resolved: ManifestNode,
  target_name: str,
  target_package: Optional[str]
  ) -> None:
@@ -468,14 +469,14 @@ def validate(
 class OperationRefResolver(RuntimeRefResolver):
  def validate(
  self,
- resolved: NonSourceNode,
+ resolved: ManifestNode,
  target_name: str,
  target_package: Optional[str],
  ) -> None:
  pass
 
  def create_relation(
- self, target_model: NonSourceNode, name: str
+ self, target_model: ManifestNode, name: str
  ) -> RelationProxy:
  if target_model.is_ephemeral_model:
  # In operations, we can't ref() ephemeral nodes, because
@@ -627,7 +628,7 @@ def __init__(
  )
  # mypy appeasement - we know it'll be a RuntimeConfig
  self.config: RuntimeConfig
- self.model: Union[ParsedMacro, NonSourceNode] = model
+ self.model: Union[ParsedMacro, ManifestNode] = model
  super().__init__(config, manifest, model.package_name)
  self.sql_results: Dict[str, AttrDict] = {}
  self.context_config: Optional[ContextConfigType] = context_config
@@ -1196,7 +1197,7 @@ def __init__(
 
 
 class ModelContext(ProviderContext):
- model: NonSourceNode
+ model: ManifestNode
 
  @contextproperty
  def pre_hooks(self) -> List[Dict[str, Any]]:
@@ -1267,7 +1268,7 @@ def this(self) -> Optional[RelationProxy]:
 
 
 def generate_parser_model(
- model: NonSourceNode,
+ model: ManifestNode,
  config: RuntimeConfig,
  manifest: Manifest,
  context_config: ContextConfigType,
@@ -1302,7 +1303,7 @@ def generate_generate_component_name_macro(
 
 
 def generate_runtime_model(
- model: NonSourceNode,
+ model: ManifestNode,
  config: RuntimeConfig,
  manifest: Manifest,
 ) -> Dict[str, Any]:
@@ -1322,3 +1323,45 @@ def generate_runtime_macro(
  macro, config, manifest, OperationProvider(), package_name
  )
  return ctx.to_dict()
+
+
+class ReportRefResolver(BaseResolver):
+ def __call__(self, *args) -> str:
+ if len(args) not in (1, 2):
+ ref_invalid_args(self.model, args)
+ self.model.refs.append(list(args))
+ return ''
+
+
+class ReportSourceResolver(BaseResolver):
+ def __call__(self, *args) -> str:
+ if len(args) != 2:
+ raise_compiler_error(
+ f"source() takes exactly two arguments ({len(args)} given)",
+ self.model
+ )
+ self.model.sources.append(list(args))
+ return ''
+
+
+def generate_parse_report(
+ report: ParsedReport,
+ config: RuntimeConfig,
+ manifest: Manifest,
+ package_name: str,
+) -> Dict[str, Any]:
+ project = config.load_dependencies()[package_name]
+ return {
+ 'ref': ReportRefResolver(
+ None,
+ report,
+ project,
+ manifest,
+ ),
+ 'source': ReportSourceResolver(
+ None,
+ report,
+ project,
+ manifest,
+ )
+ }

core/dbt/contracts/files.py

@@ -121,6 +121,7 @@ class SourceFile(JsonSchemaMixin):
  docs: List[str] = field(default_factory=list)
  macros: List[str] = field(default_factory=list)
  sources: List[str] = field(default_factory=list)
+ reports: List[str] = field(default_factory=list)
  # any node patches in this file. The entries are names, not unique ids!
  patches: List[str] = field(default_factory=list)
  # any macro patches in this file. The entries are package, name pairs.

core/dbt/contracts/graph/compiled.py

@@ -5,6 +5,7 @@
  ParsedDataTestNode,
  ParsedHookNode,
  ParsedModelNode,
+ ParsedReport,
  ParsedResource,
  ParsedRPCNode,
  ParsedSchemaTestNode,
@@ -202,7 +203,7 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
 
 
 # This is anything that can be in manifest.nodes.
-NonSourceNode = Union[
+ManifestNode = Union[
  NonSourceCompiledNode,
  NonSourceParsedNode,
 ]
@@ -211,6 +212,12 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
 # 'compile()' calls in the runner actually just return the original parsed
 # node they were given.
 CompileResultNode = Union[
- NonSourceNode,
+ ManifestNode,
  ParsedSourceDefinition,
 ]
+
+# anything that participates in the graph: sources, reports, manifest nodes
+GraphMemberNode = Union[
+ CompileResultNode,
+ ParsedReport,
+]