Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reports -> exposures #2780

Merged
merged 1 commit into from
Sep 21, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

### Features
- Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https:/fishtown-analytics/dbt/pull/2680), [#2723](https:/fishtown-analytics/dbt/pull/2723))
- Added "reports" ([#2730](https:/fishtown-analytics/dbt/issues/2730), [#2752](https:/fishtown-analytics/dbt/pull/2752))
- Added "exposures" ([#2730](https:/fishtown-analytics/dbt/issues/2730), [#2752](https:/fishtown-analytics/dbt/pull/2752), [#2777](https:/fishtown-analytics/dbt/issues/2777))

### Docs
- Add Report nodes ([docs#135](https:/fishtown-analytics/dbt-docs/issues/135), [docs#136](https:/fishtown-analytics/dbt-docs/pull/136))
Expand Down
6 changes: 3 additions & 3 deletions core/dbt/compilation.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,9 +426,9 @@ def link_graph(self, linker: Linker, manifest: Manifest):
linker.add_node(source.unique_id)
for node in manifest.nodes.values():
self.link_node(linker, node, manifest)
for report in manifest.reports.values():
self.link_node(linker, report, manifest)
# linker.add_node(report.unique_id)
for exposure in manifest.exposures.values():
self.link_node(linker, exposure, manifest)
# linker.add_node(exposure.unique_id)

cycle = linker.find_cycles()

Expand Down
18 changes: 9 additions & 9 deletions core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
)
from dbt.contracts.graph.parsed import (
ParsedMacro,
ParsedReport,
ParsedExposure,
ParsedSeedNode,
ParsedSourceDefinition,
)
Expand Down Expand Up @@ -1325,15 +1325,15 @@ def generate_runtime_macro(
return ctx.to_dict()


class ReportRefResolver(BaseResolver):
class ExposureRefResolver(BaseResolver):
def __call__(self, *args) -> str:
if len(args) not in (1, 2):
ref_invalid_args(self.model, args)
self.model.refs.append(list(args))
return ''


class ReportSourceResolver(BaseResolver):
class ExposureSourceResolver(BaseResolver):
def __call__(self, *args) -> str:
if len(args) != 2:
raise_compiler_error(
Expand All @@ -1344,23 +1344,23 @@ def __call__(self, *args) -> str:
return ''


def generate_parse_report(
report: ParsedReport,
def generate_parse_exposure(
exposure: ParsedExposure,
config: RuntimeConfig,
manifest: Manifest,
package_name: str,
) -> Dict[str, Any]:
project = config.load_dependencies()[package_name]
return {
'ref': ReportRefResolver(
'ref': ExposureRefResolver(
None,
report,
exposure,
project,
manifest,
),
'source': ReportSourceResolver(
'source': ExposureSourceResolver(
None,
report,
exposure,
project,
manifest,
)
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/contracts/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ class SourceFile(JsonSchemaMixin):
docs: List[str] = field(default_factory=list)
macros: List[str] = field(default_factory=list)
sources: List[str] = field(default_factory=list)
reports: List[str] = field(default_factory=list)
exposures: List[str] = field(default_factory=list)
# any node patches in this file. The entries are names, not unique ids!
patches: List[str] = field(default_factory=list)
# any macro patches in this file. The entries are package, name pairs.
Expand Down
6 changes: 3 additions & 3 deletions core/dbt/contracts/graph/compiled.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
ParsedDataTestNode,
ParsedHookNode,
ParsedModelNode,
ParsedReport,
ParsedExposure,
ParsedResource,
ParsedRPCNode,
ParsedSchemaTestNode,
Expand Down Expand Up @@ -216,8 +216,8 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
ParsedSourceDefinition,
]

# anything that participates in the graph: sources, reports, manifest nodes
# anything that participates in the graph: sources, exposures, manifest nodes
GraphMemberNode = Union[
CompileResultNode,
ParsedReport,
ParsedExposure,
]
26 changes: 13 additions & 13 deletions core/dbt/contracts/graph/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
)
from dbt.contracts.graph.parsed import (
ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch,
ParsedSourceDefinition, ParsedReport
ParsedSourceDefinition, ParsedExposure
)
from dbt.contracts.files import SourceFile
from dbt.contracts.util import (
Expand Down Expand Up @@ -429,7 +429,7 @@ class Manifest:
sources: MutableMapping[str, ParsedSourceDefinition]
macros: MutableMapping[str, ParsedMacro]
docs: MutableMapping[str, ParsedDocumentation]
reports: MutableMapping[str, ParsedReport]
exposures: MutableMapping[str, ParsedExposure]
generated_at: datetime
disabled: List[CompileResultNode]
files: MutableMapping[str, SourceFile]
Expand All @@ -455,7 +455,7 @@ def from_macros(
sources={},
macros=macros,
docs={},
reports={},
exposures={},
generated_at=datetime.utcnow(),
disabled=[],
files=files,
Expand All @@ -481,8 +481,8 @@ def sync_update_node(
_update_into(self.nodes, new_node)
return new_node

def update_report(self, new_report: ParsedReport):
_update_into(self.reports, new_report)
def update_exposure(self, new_exposure: ParsedExposure):
_update_into(self.exposures, new_exposure)

def update_node(self, new_node: ManifestNode):
_update_into(self.nodes, new_node)
Expand Down Expand Up @@ -725,7 +725,7 @@ def deepcopy(self):
sources={k: _deepcopy(v) for k, v in self.sources.items()},
macros={k: _deepcopy(v) for k, v in self.macros.items()},
docs={k: _deepcopy(v) for k, v in self.docs.items()},
reports={k: _deepcopy(v) for k, v in self.reports.items()},
exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
generated_at=self.generated_at,
disabled=[_deepcopy(n) for n in self.disabled],
metadata=self.metadata,
Expand All @@ -736,7 +736,7 @@ def writable_manifest(self):
edge_members = list(chain(
self.nodes.values(),
self.sources.values(),
self.reports.values(),
self.exposures.values(),
))
forward_edges, backward_edges = build_edges(edge_members)

Expand All @@ -745,7 +745,7 @@ def writable_manifest(self):
sources=self.sources,
macros=self.macros,
docs=self.docs,
reports=self.reports,
exposures=self.exposures,
generated_at=self.generated_at,
metadata=self.metadata,
disabled=self.disabled,
Expand All @@ -766,8 +766,8 @@ def expect(self, unique_id: str) -> GraphMemberNode:
return self.nodes[unique_id]
elif unique_id in self.sources:
return self.sources[unique_id]
elif unique_id in self.reports:
return self.reports[unique_id]
elif unique_id in self.exposures:
return self.exposures[unique_id]
else:
# something terrible has happened
raise dbt.exceptions.InternalException(
Expand Down Expand Up @@ -910,7 +910,7 @@ def __reduce_ex__(self, protocol):
self.sources,
self.macros,
self.docs,
self.reports,
self.exposures,
self.generated_at,
self.disabled,
self.files,
Expand Down Expand Up @@ -945,9 +945,9 @@ class WritableManifest(JsonSchemaMixin, Writable, Readable):
'The docs defined in the dbt project and its dependencies'
))
)
reports: Mapping[UniqueID, ParsedReport] = field(
exposures: Mapping[UniqueID, ParsedExposure] = field(
metadata=dict(description=(
'The reports defined in the dbt project and its dependencies'
'The exposures defined in the dbt project and its dependencies'
))
)
disabled: Optional[List[CompileResultNode]] = field(metadata=dict(
Expand Down
24 changes: 12 additions & 12 deletions core/dbt/contracts/graph/parsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
UnparsedBaseNode, FreshnessThreshold, ExternalTable,
HasYamlMetadata, MacroArgument, UnparsedSourceDefinition,
UnparsedSourceTableDefinition, UnparsedColumn, TestDef,
ReportOwner, ExposureType, MaturityType
ExposureOwner, ExposureType, MaturityType
)
from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
from dbt.exceptions import warn_or_error
Expand Down Expand Up @@ -636,11 +636,11 @@ def search_name(self):


@dataclass
class ParsedReport(UnparsedBaseNode, HasUniqueID, HasFqn):
class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
name: str
type: ExposureType
owner: ReportOwner
resource_type: NodeType = NodeType.Report
owner: ExposureOwner
resource_type: NodeType = NodeType.Exposure
maturity: Optional[MaturityType] = None
url: Optional[str] = None
description: Optional[str] = None
Expand All @@ -661,25 +661,25 @@ def search_name(self):
def tags(self):
return []

def same_depends_on(self, old: 'ParsedReport') -> bool:
def same_depends_on(self, old: 'ParsedExposure') -> bool:
return set(self.depends_on.nodes) == set(old.depends_on.nodes)

def same_description(self, old: 'ParsedReport') -> bool:
def same_description(self, old: 'ParsedExposure') -> bool:
return self.description == old.description

def same_maturity(self, old: 'ParsedReport') -> bool:
def same_maturity(self, old: 'ParsedExposure') -> bool:
return self.maturity == old.maturity

def same_owner(self, old: 'ParsedReport') -> bool:
def same_owner(self, old: 'ParsedExposure') -> bool:
return self.owner == old.owner

def same_exposure_type(self, old: 'ParsedReport') -> bool:
def same_exposure_type(self, old: 'ParsedExposure') -> bool:
return self.type == old.type

def same_url(self, old: 'ParsedReport') -> bool:
def same_url(self, old: 'ParsedExposure') -> bool:
return self.url == old.url

def same_contents(self, old: Optional['ParsedReport']) -> bool:
def same_contents(self, old: Optional['ParsedExposure']) -> bool:
# existing when it didn't before is a change!
if old is None:
return True
Expand All @@ -700,6 +700,6 @@ def same_contents(self, old: Optional['ParsedReport']) -> bool:
ParsedDocumentation,
ParsedMacro,
ParsedNode,
ParsedReport,
ParsedExposure,
ParsedSourceDefinition,
]
6 changes: 3 additions & 3 deletions core/dbt/contracts/graph/unparsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,16 +405,16 @@ class MaturityType(StrEnum):


@dataclass
class ReportOwner(JsonSchemaMixin, Replaceable):
class ExposureOwner(JsonSchemaMixin, Replaceable):
email: str
name: Optional[str] = None


@dataclass
class UnparsedReport(JsonSchemaMixin, Replaceable):
class UnparsedExposure(JsonSchemaMixin, Replaceable):
name: str
type: ExposureType
owner: ReportOwner
owner: ExposureOwner
maturity: Optional[MaturityType] = None
url: Optional[str] = None
description: Optional[str] = None
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/graph/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

INTERSECTION_DELIMITER = ','

DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'report:*']
DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'exposure:*']
DEFAULT_EXCLUDES: List[str] = []
DATA_TEST_SELECTOR: str = 'test_type:data'
SCHEMA_TEST_SELECTOR: str = 'test_type:schema'
Expand Down
6 changes: 3 additions & 3 deletions core/dbt/graph/queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import networkx as nx # type: ignore

from .graph import UniqueId
from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedReport
from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedExposure
from dbt.contracts.graph.compiled import GraphMemberNode
from dbt.contracts.graph.manifest import Manifest
from dbt.node_types import NodeType
Expand Down Expand Up @@ -50,8 +50,8 @@ def _include_in_cost(self, node_id: UniqueId) -> bool:
node = self.manifest.expect(node_id)
if node.resource_type != NodeType.Model:
return False
# must be a Model - tell mypy this won't be a Source or Report
assert not isinstance(node, (ParsedSourceDefinition, ParsedReport))
# must be a Model - tell mypy this won't be a Source or Exposure
assert not isinstance(node, (ParsedSourceDefinition, ParsedExposure))
if node.is_ephemeral:
return False
return True
Expand Down
6 changes: 3 additions & 3 deletions core/dbt/graph/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def _is_graph_member(self, unique_id: UniqueId) -> bool:
if unique_id in self.manifest.sources:
source = self.manifest.sources[unique_id]
return source.config.enabled
elif unique_id in self.manifest.reports:
elif unique_id in self.manifest.exposures:
return True
node = self.manifest.nodes[unique_id]
return not node.empty and node.config.enabled
Expand All @@ -146,8 +146,8 @@ def _is_match(self, unique_id: UniqueId) -> bool:
node = self.manifest.nodes[unique_id]
elif unique_id in self.manifest.sources:
node = self.manifest.sources[unique_id]
elif unique_id in self.manifest.reports:
node = self.manifest.reports[unique_id]
elif unique_id in self.manifest.exposures:
node = self.manifest.exposures[unique_id]
else:
raise InternalException(
f'Node {unique_id} not found in the manifest!'
Expand Down
Loading