diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 7180fb79f71..586d35ae364 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -379,10 +379,8 @@ def render_template(template, ctx, node=None): return template.render(ctx) -def get_rendered(string, ctx, node=None, - capture_macros=False): - template = get_template(string, ctx, node, - capture_macros=capture_macros) +def get_rendered(string, ctx, node=None, capture_macros=False): + template = get_template(string, ctx, node, capture_macros=capture_macros) return render_template(template, ctx, node) diff --git a/core/dbt/context/docs.py b/core/dbt/context/docs.py index 6bed0600b09..b050f82ec48 100644 --- a/core/dbt/context/docs.py +++ b/core/dbt/context/docs.py @@ -1,5 +1,5 @@ from typing import ( - Any, Optional, List, Dict, Union + Any, Dict, Union ) from dbt.exceptions import ( @@ -9,7 +9,7 @@ from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.compiled import CompileResultNode from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.parsed import Docref, ParsedMacro +from dbt.contracts.graph.parsed import ParsedMacro from dbt.context.base import contextmember from dbt.context.configured import ConfiguredContext @@ -20,29 +20,15 @@ def __init__( self, config: RuntimeConfig, node: Any, - docrefs: List[Docref], - column_name: Optional[str], ) -> None: super().__init__(config) self.node = node - self.docrefs = docrefs - self.column_name = column_name @contextmember def doc(self, *args: str) -> str: # when you call doc(), this is what happens at parse time if len(args) != 1 and len(args) != 2: doc_invalid_args(self.node, args) - doc_package_name = '' - doc_name = args[0] - if len(args) == 2: - doc_package_name = args[1] - - docref = Docref(documentation_package=doc_package_name, - documentation_name=doc_name, - column_name=self.column_name) - self.docrefs.append(docref) - # At parse time, nothing should care about what doc() returns return '' @@ -87,11 +73,8 @@ def doc(self, *args: str) -> str: def generate_parser_docs( config: RuntimeConfig, unparsed: Any, - docrefs: List[Docref], - column_name: Optional[str] = None, ) -> Dict[str, Any]: - - ctx = DocsParseContext(config, unparsed, docrefs, column_name) + ctx = DocsParseContext(config, unparsed) return ctx.to_dict() diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 9695ebaec25..6fe9542a404 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -23,7 +23,7 @@ from dbt.contracts.graph.unparsed import ( UnparsedNode, UnparsedMacro, UnparsedDocumentationFile, Quoting, UnparsedBaseNode, FreshnessThreshold, ExternalTable, - AdditionalPropertiesAllowed, HasYamlMetadata + AdditionalPropertiesAllowed, HasYamlMetadata, MacroArgument ) from dbt.contracts.util import Replaceable, list_str from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -125,17 +125,6 @@ class ColumnInfo(JsonSchemaMixin, Replaceable): tags: List[str] = field(default_factory=list) -# Docrefs are not quite like regular references, as they indicate what they -# apply to as well as what they are referring to (so the doc package + doc -# name, but also the column name if relevant). This is because column -# descriptions are rendered separately from their models. -@dataclass -class Docref(JsonSchemaMixin, Replaceable): - documentation_name: str - documentation_package: str - column_name: Optional[str] = None - - @dataclass class HasFqn(JsonSchemaMixin, Replaceable): fqn: List[str] @@ -186,7 +175,6 @@ def patch(self, patch: 'ParsedNodePatch'): self.patch_path: Optional[str] = patch.original_file_path self.description = patch.description self.columns = patch.columns - self.docrefs = patch.docrefs self.meta = patch.meta if dbt.flags.STRICT_MODE: assert isinstance(self, JsonSchemaMixin) @@ -221,7 +209,6 @@ class ParsedNodeDefaults(ParsedNodeMandatory): refs: List[List[str]] = field(default_factory=list) sources: List[List[Any]] = field(default_factory=list) depends_on: DependsOn = field(default_factory=DependsOn) - docrefs: List[Docref] = field(default_factory=list) description: str = field(default='') columns: Dict[str, ColumnInfo] = field(default_factory=dict) meta: Dict[str, Any] = field(default_factory=dict) @@ -467,7 +454,6 @@ def json_schema(cls, embeddable: bool = False) -> Dict[str, Any]: class ParsedPatch(HasYamlMetadata, Replaceable): name: str description: str - docrefs: List[Docref] meta: Dict[str, Any] @@ -481,7 +467,7 @@ class ParsedNodePatch(ParsedPatch): @dataclass class ParsedMacroPatch(ParsedPatch): - pass + arguments: List[MacroArgument] = field(default_factory=list) @dataclass @@ -497,10 +483,10 @@ class ParsedMacro(UnparsedMacro, HasUniqueID): tags: List[str] = field(default_factory=list) # TODO: is this ever populated? depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) - docrefs: List[Docref] = field(default_factory=list) - description: str = field(default='') + description: str = '' meta: Dict[str, Any] = field(default_factory=dict) patch_path: Optional[str] = None + arguments: List[MacroArgument] = field(default_factory=list) def local_vars(self): return {} @@ -515,8 +501,8 @@ def generator(self) -> MacroGenerator: def patch(self, patch: ParsedMacroPatch): self.patch_path: Optional[str] = patch.original_file_path self.description = patch.description - self.docrefs = patch.docrefs self.meta = patch.meta + self.arguments = patch.arguments if dbt.flags.STRICT_MODE: assert isinstance(self, JsonSchemaMixin) self.to_dict(validate=True) @@ -548,7 +534,6 @@ class ParsedSourceDefinition( loaded_at_field: Optional[str] = None freshness: Optional[FreshnessThreshold] = None external: Optional[ExternalTable] = None - docrefs: List[Docref] = field(default_factory=list) description: str = '' columns: Dict[str, ColumnInfo] = field(default_factory=dict) meta: Dict[str, Any] = field(default_factory=dict) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index c34b2758cd9..852709224bd 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -110,9 +110,16 @@ class UnparsedNodeUpdate(HasColumnTests, HasTests, HasYamlMetadata): pass +@dataclass +class MacroArgument(JsonSchemaMixin): + name: str + type: Optional[str] = None + description: str = '' + + @dataclass class UnparsedMacroUpdate(HasDocs, HasYamlMetadata): - pass + arguments: List[MacroArgument] = field(default_factory=list) class TimePeriod(StrEnum): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 2988ad2d599..75f9084b207 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -10,6 +10,7 @@ from dbt.logger import GLOBAL_LOGGER as logger, DbtProcessState from dbt.node_types import NodeType +from dbt.clients.jinja import get_rendered from dbt.clients.system import make_directory from dbt.config import Project, RuntimeConfig from dbt.context.docs import generate_runtime_docs @@ -445,18 +446,9 @@ def _process_docs_for_node( context: Dict[str, Any], node: NonSourceNode, ): - for docref in node.docrefs: - column_name = docref.column_name - - if column_name is None: - obj = node - else: - obj = _get_node_column(node, column_name) - - raw = obj.description or '' - # At this point, we know that our documentation string has a - # 'docs("...")' pointing at it. We want to render it. - obj.description = dbt.clients.jinja.get_rendered(raw, context) + node.description = get_rendered(node.description, context) + for column_name, column in node.columns.items(): + column.description = get_rendered(column.description, context) def _process_docs_for_source( @@ -465,25 +457,23 @@ def _process_docs_for_source( ): table_description = source.description source_description = source.source_description - table_description = dbt.clients.jinja.get_rendered(table_description, - context) - source_description = dbt.clients.jinja.get_rendered(source_description, - context) + table_description = get_rendered(table_description, context) + source_description = get_rendered(source_description, context) source.description = table_description source.source_description = source_description for column in source.columns.values(): column_desc = column.description - column_desc = dbt.clients.jinja.get_rendered(column_desc, context) + column_desc = get_rendered(column_desc, context) column.description = column_desc def _process_docs_for_macro( context: Dict[str, Any], macro: ParsedMacro ) -> None: - for docref in macro.docrefs: - raw = macro.description or '' - macro.description = dbt.clients.jinja.get_rendered(raw, context) + macro.description = get_rendered(macro.description, context) + for arg in macro.arguments: + arg.description = get_rendered(arg.description, context) def process_docs(manifest: Manifest, config: RuntimeConfig): diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index f23a627ca4a..639eae3d3cd 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -19,7 +19,6 @@ ParsedNodePatch, ParsedSourceDefinition, ColumnInfo, - Docref, ParsedTestNode, ParsedMacroPatch, ) @@ -76,7 +75,6 @@ class ParserRef: """A helper object to hold parse-time references.""" def __init__(self): self.column_info: Dict[str, ColumnInfo] = {} - self.docrefs: List[Docref] = [] def add(self, column: UnparsedColumn, description, data_type, meta): self.column_info[column.name] = ColumnInfo( @@ -88,14 +86,12 @@ def add(self, column: UnparsedColumn, description, data_type, meta): ) -def collect_docrefs( +def column_info( config: RuntimeConfig, target: UnparsedSchemaYaml, - refs: ParserRef, - column_name: Optional[str], *descriptions: str, ) -> None: - context = generate_parser_docs(config, target, refs.docrefs, column_name) + context = generate_parser_docs(config, target) for description in descriptions: get_rendered(description, context) @@ -360,15 +356,12 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: def parse_docs(self, block: TargetBlock) -> ParserRef: refs = ParserRef() for column in block.columns: - column_name = column.name description = column.description data_type = column.data_type meta = column.meta - collect_docrefs( + column_info( self.root_project, block.target, - refs, - column_name, description, ) @@ -446,12 +439,11 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: def parse_docs(self, block: TargetBlock) -> ParserRef: refs = ParserRef() for column in block.columns: - column_name = column.name description = column.description data_type = column.data_type meta = column.meta - collect_docrefs( - self.root_project, block.target, refs, column_name, description + column_info( + self.root_project, block.target, description ) refs.add(column, description, data_type, meta) @@ -533,9 +525,8 @@ def parse_patch( description = table.description or '' meta = table.meta or {} source_description = source.description or '' - collect_docrefs( - self.root_project, source, refs, None, description, - source_description + column_info( + self.root_project, source, description, source_description ) loaded_at_field = table.loaded_at_field or source.loaded_at_field @@ -566,7 +557,6 @@ def parse_patch( source_meta=source_meta, meta=meta, loader=source.loader, - docrefs=refs.docrefs, loaded_at_field=loaded_at_field, freshness=freshness, quoting=quoting, @@ -580,12 +570,12 @@ def parse_patch( class NonSourceParser( YamlDocsReader[NonSourceTarget, Parsed], Generic[NonSourceTarget, Parsed] ): - def collect_docrefs( - self, block: TargetBlock[NonSourceTarget], refs: ParserRef + def collect_column_info( + self, block: TargetBlock[NonSourceTarget] ) -> str: description = block.target.description - collect_docrefs( - self.root_project, block.target, refs, None, description + column_info( + self.root_project, block.target, description ) return description @@ -618,7 +608,7 @@ class NodePatchParser( def parse_patch( self, block: TargetBlock[NodeTarget], refs: ParserRef ) -> None: - description = self.collect_docrefs(block, refs) + description = self.collect_column_info(block) result = ParsedNodePatch( name=block.target.name, original_file_path=block.target.original_file_path, @@ -626,7 +616,6 @@ def parse_patch( package_name=block.target.package_name, description=description, columns=refs.column_info, - docrefs=refs.docrefs, meta=block.target.meta, ) self.results.add_patch(self.yaml.file, result) @@ -649,6 +638,16 @@ def _target_type(self) -> Type[UnparsedAnalysisUpdate]: class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]): + def collect_column_info( + self, block: TargetBlock[UnparsedMacroUpdate] + ) -> str: + description = block.target.description + arg_docs = [arg.description for arg in block.target.arguments] + column_info( + self.root_project, block.target, description, *arg_docs + ) + return description + def get_block(self, node: UnparsedMacroUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) @@ -658,15 +657,15 @@ def _target_type(self) -> Type[UnparsedMacroUpdate]: def parse_patch( self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef ) -> None: - description = self.collect_docrefs(block, refs) + description = self.collect_column_info(block) result = ParsedMacroPatch( name=block.target.name, original_file_path=block.target.original_file_path, yaml_key=block.target.yaml_key, package_name=block.target.package_name, + arguments=block.target.arguments, description=description, - docrefs=refs.docrefs, meta=block.target.meta, ) self.results.add_macro_patch(self.yaml.file, result) diff --git a/test/integration/029_docs_generate_tests/ref_models/docs.md b/test/integration/029_docs_generate_tests/ref_models/docs.md index 61200373fba..595444ec933 100644 --- a/test/integration/029_docs_generate_tests/ref_models/docs.md +++ b/test/integration/029_docs_generate_tests/ref_models/docs.md @@ -29,3 +29,7 @@ An ID field {% docs macro_info %} My custom test that I wrote that does nothing {% enddocs %} + +{% docs macro_arg_info %} +The model for my custom test +{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/ref_models/schema.yml b/test/integration/029_docs_generate_tests/ref_models/schema.yml index d0350ec4ff5..997eccb60c5 100644 --- a/test/integration/029_docs_generate_tests/ref_models/schema.yml +++ b/test/integration/029_docs_generate_tests/ref_models/schema.yml @@ -35,3 +35,7 @@ macros: description: "{{ doc('macro_info') }}" meta: some_key: 100 + arguments: + - name: model + type: Relation + description: "{{ doc('macro_arg_info') }}" diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 91efac1617c..90d8f033aa7 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -833,7 +833,7 @@ def _verify_generic_macro_structure(self, manifest): { 'path', 'original_file_path', 'package_name', 'raw_sql', 'root_path', 'name', 'unique_id', 'tags', 'resource_type', - 'depends_on', 'meta', 'description', 'patch_path', 'docrefs', + 'depends_on', 'meta', 'description', 'patch_path', 'arguments' } ) # Don't compare the sql, just make sure it exists @@ -857,8 +857,8 @@ def _verify_generic_macro_structure(self, manifest): 'depends_on': {'macros': []}, 'description': '', 'patch_path': None, - 'docrefs': [], 'meta': {}, + 'arguments': [], }, without_sql, ) @@ -950,7 +950,6 @@ def expected_seeded_manifest(self, model_database=None): }, }, 'patch_path': model_schema_yml_path, - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -1032,7 +1031,6 @@ def expected_seeded_manifest(self, model_database=None): 'tags': [], }, }, - 'docrefs': [], 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, @@ -1075,7 +1073,6 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.not_null_model_id', - 'docrefs': [], 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, @@ -1123,7 +1120,6 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.test_nothing_model_', - 'docrefs': [], 'compiled': True, 'compiled_sql': AnyStringWith('select 0'), 'extra_ctes_injected': True, @@ -1171,7 +1167,6 @@ def expected_seeded_manifest(self, model_database=None): 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.unique_model_id', - 'docrefs': [], 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, @@ -1306,6 +1301,9 @@ def expected_postgres_references_manifest(self, model_database=None): macro_info = LineIndifferent( '{% docs macro_info %}\nMy custom test that I wrote that does nothing\n{% enddocs %}' ) + macro_arg_info = LineIndifferent( + '{% docs macro_arg_info %}\nThe model for my custom test\n{% enddocs %}' + ) return { 'nodes': { @@ -1330,7 +1328,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['source.test.my_source.my_table'] }, 'description': '', - 'docrefs': [], 'fqn': ['test', 'ephemeral_copy'], 'name': 'ephemeral_copy', 'original_file_path': self.dir('ref_models/ephemeral_copy.sql'), @@ -1392,23 +1389,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['model.test.ephemeral_copy'] }, 'description': 'A summmary table of the ephemeral copy of the seed data', - 'docrefs': [ - { - 'column_name': 'first_name', - 'documentation_name': 'summary_first_name', - 'documentation_package': '' - }, - { - 'column_name': 'ct', - 'documentation_name': 'summary_count', - 'documentation_package': '' - }, - { - 'column_name': None, - 'documentation_name': 'ephemeral_summary', - 'documentation_package': '' - } - ], 'fqn': ['test', 'ephemeral_summary'], 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), @@ -1472,23 +1452,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'nodes': ['model.test.ephemeral_summary'] }, 'description': 'A view of the summary of the ephemeral copy of the seed data', - 'docrefs': [ - { - 'column_name': 'first_name', - 'documentation_name': 'summary_first_name', - 'documentation_package': '' - }, - { - 'column_name': 'ct', - 'documentation_name': 'summary_count', - 'documentation_package': '' - }, - { - 'column_name': None, - 'documentation_name': 'view_summary', - 'documentation_package': '' - } - ], 'fqn': ['test', 'view_summary'], 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), @@ -1570,7 +1533,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', - 'docrefs': [], 'fqn': ['test', 'seed'], 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), @@ -1611,23 +1573,6 @@ def expected_postgres_references_manifest(self, model_database=None): }, 'database': self.default_database, 'description': 'My table', - 'docrefs': [ - { - 'documentation_package': '', - 'documentation_name': 'column_info', - 'column_name': 'id', - }, - { - 'column_name': None, - 'documentation_name': 'table_info', - 'documentation_package': '', - }, - { - 'column_name': None, - 'documentation_name': 'source_info', - 'documentation_package': '', - }, - ], 'external': { 'file_format': None, 'location': None, 'partitions': None, 'row_format': None, 'tbl_properties': None @@ -1739,6 +1684,16 @@ def expected_postgres_references_manifest(self, model_database=None): 'root_path': self.test_root_dir, 'unique_id': 'test.macro_info', }, + 'test.macro_arg_info': { + 'block_contents': 'The model for my custom test', + 'file_contents': macro_arg_info, + 'name': 'macro_arg_info', + 'original_file_path': docs_path, + 'package_name': 'test', + 'path': 'docs.md', + 'root_path': self.test_root_dir, + 'unique_id': 'test.macro_arg_info', + }, }, 'child_map': { 'model.test.ephemeral_copy': ['model.test.ephemeral_summary'], @@ -1825,6 +1780,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'test.table_info', 'test.column_info', 'test.macro_info', + 'test.macro_arg_info', ], 'macros': [], 'nodes': [], @@ -1863,13 +1819,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': self.dir('macros/dummy_test.sql'), 'path': self.dir('macros/dummy_test.sql'), 'package_name': 'test', - 'docrefs': [ - { - 'column_name': None, - 'documentation_name': 'macro_info', - 'documentation_package': '', - }, - ], 'meta': { 'some_key': 100, }, @@ -1878,6 +1827,13 @@ def expected_postgres_references_manifest(self, model_database=None): 'unique_id': 'macro.test.test_nothing', 'tags': [], 'root_path': self.test_root_dir, + 'arguments': [ + { + 'name': 'model', + 'type': 'Relation', + 'description': 'The model for my custom test', + }, + ], } } } @@ -1962,7 +1918,6 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'A clustered and partitioned copy of the test model', 'patch_path': self.dir('bq_models/schema.yml'), - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2041,7 +1996,6 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', 'patch_path': self.dir('bq_models/schema.yml'), - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2121,7 +2075,6 @@ def expected_bigquery_complex_manifest(self): }, 'description': 'The test model', 'patch_path': self.dir('bq_models/schema.yml'), - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2165,7 +2118,6 @@ def expected_bigquery_complex_manifest(self): 'unique_id': 'model.test.nested_table', 'columns': {}, 'description': '', - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2247,7 +2199,6 @@ def expected_bigquery_complex_manifest(self): }, }, 'description': 'The test seed', - 'docrefs': [], 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, @@ -2471,7 +2422,6 @@ def expected_redshift_incremental_view_manifest(self): }, }, 'patch_path': self.dir('rs_models/schema.yml'), - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2553,7 +2503,6 @@ def expected_redshift_incremental_view_manifest(self): }, }, 'description': 'The test seed', - 'docrefs': [], 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, @@ -2779,7 +2728,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'nodes': ['seed.test.seed'] }, 'description': 'The test model', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'model'], @@ -2867,7 +2815,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], @@ -2921,7 +2868,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'not_null_model_id'], @@ -2979,7 +2925,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'database': self.default_database, 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'test_nothing_model_'], @@ -3037,7 +2982,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'database': self.default_database, 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'unique_model_id'], @@ -3142,23 +3086,6 @@ def expected_postgres_references_run_results(self): 'description': ( 'A summmary table of the ephemeral copy of the seed data' ), - 'docrefs': [ - { - 'column_name': 'first_name', - 'documentation_name': 'summary_first_name', - 'documentation_package': '' - }, - { - 'column_name': 'ct', - 'documentation_name': 'summary_count', - 'documentation_package': '' - }, - { - 'column_name': None, - 'documentation_name': 'ephemeral_summary', - 'documentation_package': '' - } - ], 'extra_ctes': [ {'id': 'model.test.ephemeral_copy', 'sql': cte_sql}, ], @@ -3240,23 +3167,6 @@ def expected_postgres_references_run_results(self): 'A view of the summary of the ephemeral copy of the ' 'seed data' ), - 'docrefs': [ - { - 'column_name': 'first_name', - 'documentation_name': 'summary_first_name', - 'documentation_package': '' - }, - { - 'column_name': 'ct', - 'documentation_name': 'summary_count', - 'documentation_package': '' - }, - { - 'column_name': None, - 'documentation_name': 'view_summary', - 'documentation_package': '' - } - ], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], @@ -3348,7 +3258,6 @@ def expected_postgres_references_run_results(self): 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', - 'docrefs': [], 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 8375c4226b8..4b30921dd59 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -387,7 +387,7 @@ def test_model_runtime_context(config, manifest, get_adapter): def test_docs_parse_context(config): - ctx = docs.generate_parser_docs(config, mock_model(), []) + ctx = docs.generate_parser_docs(config, mock_model()) assert_has_keys(REQUIRED_DOCS_KEYS, MAYBE_KEYS, ctx) diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index 9231b0a182b..2bb95fefe97 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -44,7 +44,6 @@ def test_basic_uncompiled(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, 'compiled': False, @@ -128,7 +127,6 @@ def test_basic_compiled(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, 'compiled': True, @@ -241,7 +239,6 @@ def test_basic_uncompiled(self): 'vars': {}, 'severity': 'error', }, - 'docrefs': [], 'columns': {}, 'meta': {}, 'compiled': False, @@ -326,7 +323,6 @@ def test_basic_compiled(self): 'vars': {}, 'severity': 'warn', }, - 'docrefs': [], 'columns': {}, 'meta': {}, 'compiled': True, diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 384892d7582..27ab366f3c3 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -3,12 +3,12 @@ from dbt.node_types import NodeType from dbt.contracts.graph.parsed import ( ParsedModelNode, DependsOn, NodeConfig, ColumnInfo, Hook, ParsedTestNode, - TestConfig, ParsedSnapshotNode, TimestampSnapshotConfig, All, Docref, + TestConfig, ParsedSnapshotNode, TimestampSnapshotConfig, All, GenericSnapshotConfig, CheckSnapshotConfig, SnapshotStrategy, IntermediateSnapshotNode, ParsedNodePatch, ParsedMacro, MacroDependsOn, ParsedSourceDefinition, ParsedDocumentation, ParsedHookNode ) -from dbt.contracts.graph.unparsed import Quoting, FreshnessThreshold +from dbt.contracts.graph.unparsed import Quoting from hologram import ValidationError from .utils import ContractTestCase @@ -89,7 +89,6 @@ def test_ok(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -168,7 +167,6 @@ def test_complex(self): 'tags': [], 'vars': {'foo': 100}, }, - 'docrefs': [], 'columns': { 'a': { 'name': 'a', @@ -243,7 +241,6 @@ def test_invalid_bad_tags(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -280,7 +277,6 @@ def test_invalid_bad_materialized(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -315,9 +311,6 @@ def test_patch_ok(self): description='The foo model', original_file_path='/path/to/schema.yml', columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, - docrefs=[ - Docref(documentation_name='foo', documentation_package='test'), - ], meta={}, ) @@ -362,12 +355,6 @@ def test_patch_ok(self): 'tags': [], }, }, - 'docrefs': [ - { - 'documentation_name': 'foo', - 'documentation_package': 'test', - } - ], } expected = self.ContractType( @@ -392,9 +379,6 @@ def test_patch_ok(self): config=NodeConfig(), patch_path='/path/to/schema.yml', columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, - docrefs=[ - Docref(documentation_name='foo', documentation_package='test'), - ], ) self.assert_symmetric(expected, expected_dict) # sanity check self.assertEqual(initial, expected) @@ -429,7 +413,6 @@ def patch_invalid(self): description=None, original_file_path='/path/to/schema.yml', columns={}, - docrefs=[], ) with self.assertRaises(ValidationError): initial.patch(patch) @@ -468,7 +451,6 @@ def test_ok(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -545,7 +527,6 @@ def test_complex(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': { 'a': { 'name': 'a', @@ -620,7 +601,6 @@ def test_invalid_index_type(self): 'tags': [], 'vars': {}, }, - 'docrefs': [], 'columns': {}, 'meta': {}, 'index': 'a string!?', @@ -663,7 +643,6 @@ def test_ok(self): 'vars': {}, 'severity': 'error', }, - 'docrefs': [], 'columns': {}, } node = self.ContractType( @@ -744,7 +723,6 @@ def test_complex(self): 'severity': 'WARN', 'extra_key': 'extra value' }, - 'docrefs': [], 'columns': { 'a': { 'name': 'a', @@ -821,7 +799,6 @@ def test_invalid_column_name_type(self): 'vars': {}, 'severity': 'ERROR', }, - 'docrefs': [], 'columns': {}, 'column_name': {}, 'meta': {}, @@ -860,7 +837,6 @@ def test_invalid_missing_severity(self): 'vars': {}, 'severtiy': 'WARN', }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -897,7 +873,6 @@ def test_invalid_severity(self): 'vars': {}, 'severity': 'WERROR', # invalid severity }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -1163,7 +1138,6 @@ def test_timestamp_ok(self): 'strategy': 'timestamp', 'updated_at': 'last_update', }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -1269,7 +1243,6 @@ def test_check_ok(self): 'strategy': 'check', 'check_cols': 'all', }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -1373,7 +1346,6 @@ def test_invalid_bad_resource_type(self): 'strategy': 'timestamp', 'updated_at': 'last_update', }, - 'docrefs': [], 'columns': {}, 'meta': {}, } @@ -1389,7 +1361,6 @@ def test_empty(self): 'description': 'The foo model', 'original_file_path': '/path/to/schema.yml', 'columns': {}, - 'docrefs': [], 'meta': {}, 'yaml_key': 'models', 'package_name': 'test', @@ -1401,7 +1372,6 @@ def test_empty(self): package_name='test', original_file_path='/path/to/schema.yml', columns={}, - docrefs=[], meta={}, ) self.assert_symmetric(patch, dct) @@ -1419,12 +1389,6 @@ def test_populated(self): 'tags': [], }, }, - 'docrefs': [ - { - 'documentation_name': 'foo', - 'documentation_package': 'test', - } - ], 'meta': {'key': ['value']}, 'yaml_key': 'models', 'package_name': 'test', @@ -1434,9 +1398,6 @@ def test_populated(self): description='The foo model', original_file_path='/path/to/schema.yml', columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, - docrefs=[ - Docref(documentation_name='foo', documentation_package='test'), - ], meta={'key': ['value']}, yaml_key='models', package_name='test', @@ -1462,7 +1423,7 @@ def test_ok(self): 'depends_on': {'macros': []}, 'meta': {}, 'description': 'my macro description', - 'docrefs': [], + 'arguments': [], } macro = ParsedMacro( name='foo', @@ -1477,7 +1438,7 @@ def test_ok(self): depends_on=MacroDependsOn(), meta={}, description='my macro description', - docrefs=[], + arguments=[], ) self.assert_symmetric(macro, macro_dict) self.assertEqual(macro.local_vars(), {}) @@ -1493,7 +1454,10 @@ def test_invalid_missing_unique_id(self): 'root_path': '/root/', 'resource_type': 'macro', 'tags': [], - 'depends_on': {'macros': []} + 'depends_on': {'macros': []}, + 'meta': {}, + 'description': 'my macro description', + 'arguments': [], } self.assert_fails_validation(bad_missing_uid) @@ -1509,6 +1473,9 @@ def test_invalid_extra_field(self): 'unique_id': 'macro.test.foo', 'tags': [], 'depends_on': {'macros': []}, + 'meta': {}, + 'description': 'my macro description', + 'arguments': [], 'extra': 'too many fields' } self.assert_fails_validation(bad_extra_field) @@ -1589,7 +1556,6 @@ def test_basic(self): 'identifier': 'my_source_table', 'resource_type': str(NodeType.Source), 'description': '', - 'docrefs': [], 'columns': {}, 'quoting': {}, 'unique_id': 'test.source.my_source.my_source_table', @@ -1599,7 +1565,6 @@ def test_basic(self): } source_def = self.ContractType( columns={}, - docrefs=[], database='some_db', description='', fqn=['test', 'source', 'my_source', 'my_source_table'], diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 251d93ec6f8..ee08b23add4 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -29,7 +29,7 @@ 'alias', 'tags', 'config', 'unique_id', 'refs', 'sources', 'meta', 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'root_path', 'path', 'original_file_path', 'raw_sql', - 'docrefs', 'description', 'columns', 'fqn', 'build_path', 'patch_path', + 'description', 'columns', 'fqn', 'build_path', 'patch_path', }) REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | { diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 21810d3ced0..3e946cd5247 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -311,7 +311,6 @@ def test__parse_basic_model_tests(self): name='my_model', description='A description of my model', columns={'color': ColumnInfo(name='color', description='The color value')}, - docrefs=[], original_file_path=normalize('models/test_one.yml'), meta={}, yaml_key='models', @@ -692,16 +691,6 @@ def setUp(self): x_uid = 'model.project.x' y_uid = 'model.otherproject.y' src_uid = 'source.thirdproject.src.tbl' - remote_docref = mock.MagicMock( - documentation_package='otherproject', - documentation_name='my_doc', - column_name=None, - ) - docref = mock.MagicMock( - documentation_package='', - documentation_name='my_doc', - column_name=None, - ) self.x_node = mock.MagicMock( __class__=ParsedModelNode, package_name='project', @@ -709,7 +698,6 @@ def setUp(self): config=mock.MagicMock(enabled=True), refs=[], sources=[['src', 'tbl']], - docrefs=[remote_docref], unique_id=x_uid, resource_type=NodeType.Model, depends_on=x_depends_on, @@ -722,7 +710,6 @@ def setUp(self): config=mock.MagicMock(enabled=True), refs=[['x']], sources=[], - docrefs=[docref], unique_id=y_uid, resource_type=NodeType.Model, depends_on=y_depends_on,