SPIKE: external tables as model materialization

dbt/adapters/base/relation.py

@@ -351,6 +351,14 @@ def is_view(self) -> bool:
  def is_materialized_view(self) -> bool:
  return self.type == RelationType.MaterializedView
 
+ @property
+ def is_external(self) -> bool:
+ return self.type == RelationType.External
+
+ @property
+ def is_external_table(self) -> bool:
+ return self.type == RelationType.ExternalTable
+
  @classproperty
  def Table(cls) -> str:
  return str(RelationType.Table)
@@ -367,6 +375,10 @@ def View(cls) -> str:
  def External(cls) -> str:
  return str(RelationType.External)
 
+ @classproperty
+ def ExternalTable(cls) -> str:
+ return str(RelationType.ExternalTable)
+
  @classproperty
  def MaterializedView(cls) -> str:
  return str(RelationType.MaterializedView)

dbt/adapters/contracts/relation.py

@@ -20,6 +20,7 @@ class RelationType(StrEnum):
  MaterializedView = "materialized_view"
  External = "external"
  Ephemeral = "ephemeral"
+ ExternalTable = "external_table"
 
 
 class MaterializationContract(Protocol):

dbt/include/global_project/macros/materializations/external_table/external_table.sql

@@ -0,0 +1,58 @@
+{% materialization external_table, default %}
+
+ {%- set identifier = model['alias'] -%}
+ {%- set full_refresh_mode = (should_full_refresh()) -%}
+
+ {%- set existing_relation = load_cached_relation(this) -%}
+ {%- set target_relation = this.incorporate(type=this.ExternalTable) %}
+
+ {{ log('existing_relation: ' ~ existing_relation, info=True) }}
+ {{ log('existing_relation.type: ' ~ existing_relation.type, info=True) }}
+
+ {%- set exists_as_table = (existing_relation is not none and existing_relation.is_table) -%}
+ {%- set exists_as_view = (existing_relation is not none and existing_relation.is_view) -%}
+ {%- set exists_as_external_table = (existing_relation is not none and existing_relation.is_external_table) -%}
+
+ -- build model
+ {% set build_plan = [] %}
+ {% set code = 'CREATE' %}
+
+ {% if exists_as_view %}
+ {% set build_plan = build_plan + [
+ drop_relation_if_exists(existing_relation),
+ create_external_table(target_relation, model.columns.values())
+ ] %}
+ {% elif exists_as_table %}
+ {% if full_refresh_mode %}
+ {% set build_plan = build_plan + [create_external_table(target_relation, model.columns.values())] %}
+ {% elif not full_refresh_mode %} 
+ {% set code = 'REFRESH' %}
+ {% set build_plan = build_plan + refresh_external_table(target_relation) %}
+ {% endif %}
+ {% else %}
+ {% set build_plan = build_plan + [
+ create_external_table(target_relation, model.columns.values())
+ ] %}
+ {% endif %}
+
+ {% call noop_statement('main', code, code) %}
+ {{ build_plan }};
+ {% endcall %}
+
+ {%- set grant_config = config.get('grants') -%}
+
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+ {% do persist_docs(target_relation, model) %}
+
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+ -- `COMMIT` happens here
+ {{ adapter.commit() }}
+
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+ {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}

dbt/include/global_project/macros/materializations/external_table/get_create_external_table.sql

@@ -0,0 +1,7 @@
+{% macro create_external_table(relation, columns) %}
+ {{ adapter.dispatch('create_external_table', 'dbt')(relation, columns) }}
+{% endmacro %}
+
+{% macro default__create_external_table(relation, columns) %}
+ {{ exceptions.raise_compiler_error("External table creation is not implemented for the default adapter") }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/external_table/helpers.sql

@@ -0,0 +1,54 @@
+{% macro refresh_external_table(source_node) %}
+ {{ return(adapter.dispatch('refresh_external_table', 'dbt')(source_node)) }}
+{% endmacro %}
+
+{% macro default__refresh_external_table(source_node) %}
+ {% do return([]) %}
+{% endmacro %}
+
+{% macro update_external_table_columns(source_node) %}
+ {{ return(adapter.dispatch('update_external_table_columns', 'dbt')(source_node)) }}
+{% endmacro %}
+
+{% macro default__update_external_table_columns(source_node) %}
+
+{% endmacro %}
+
+{%- macro create_external_schema(source_node) -%}
+ {{ adapter.dispatch('create_external_schema', 'dbt')(source_node) }}
+{%- endmacro -%}
+
+{%- macro default__create_external_schema(source_node) -%}
+ {%- set fqn -%}
+ {%- if source_node.database -%}
+ {{ source_node.database }}.{{ source_node.schema }}
+ {%- else -%}
+ {{ source_node.schema }}
+ {%- endif -%}
+ {%- endset -%}
+
+ {%- set ddl -%}
+ create schema if not exists {{ fqn }}
+ {%- endset -%}
+
+ {{ return(ddl) }}
+{%- endmacro -%}
+
+
+{% macro exit_transaction() %}
+ {{ return(adapter.dispatch('exit_transaction', 'dbt')()) }}
+{% endmacro %}
+
+{% macro default__exit_transaction() %}
+ {{ return('') }}
+{% endmacro %}
+
+{% macro dropif(node) %}
+ {{ adapter.dispatch('dropif', 'dbt')(node) }}
+{% endmacro %}
+
+{% macro default__dropif() %}
+ {{ exceptions.raise_compiler_error(
+ "Dropping external tables is not implemented for the default adapter"
+ ) }}
+{% endmacro %}