Skip to content

Commit

Permalink
Fix incremental delete+insert SQL
Browse files Browse the repository at this point in the history
resolves dbt-labs#150

Problem
The delete query for the 'delete+insert' incremental_strategy with 2+ unique_key columns is VERY inefficient. In many cases, it will hang and never return for deleting small amounts of data (<100K rows).

Solution
Improve the query by switching to a much more efficient delete strategy:

```
delete from table1
where (col1, col2) in (
    select distinct col1, col2 from table1_tmp
)
```
  • Loading branch information
ataft committed Apr 10, 2024
1 parent 7392deb commit 773e8d3
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 27 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20240410-163601.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Fix incremental delete+insert SQL
time: 2024-04-10T16:36:43.253228-05:00
custom:
Author: ataft
Issue: "150"
Original file line number Diff line number Diff line change
Expand Up @@ -61,34 +61,23 @@
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}

{% if unique_key %}
{% if unique_key is sequence and unique_key is not string %}
delete from {{target }}
using {{ source }}
where (
{% for key in unique_key %}
{{ source }}.{{ key }} = {{ target }}.{{ key }}
{{ "and " if not loop.last}}
{% endfor %}
{% if incremental_predicates %}
{% for predicate in incremental_predicates %}
and {{ predicate }}
{% endfor %}
{% endif %}
);
{% else %}
delete from {{ target }}
where (
{{ unique_key }}) in (
select ({{ unique_key }})
from {{ source }}
)
{%- if incremental_predicates %}
{% for predicate in incremental_predicates %}
and {{ predicate }}
{% endfor %}
{%- endif -%};

{% if unique_key is string %}
{% set unique_key = [unique_key] %}
{% endif %}

{%- set unique_key_str = unique_key|join(', ') -%}

delete from {{ target }}
where ({{ unique_key_str }}) in (
select distinct {{ unique_key_str }}
from {{ source }}
)
{%- if incremental_predicates %}
{% for predicate in incremental_predicates %}
and {{ predicate }}
{% endfor %}
{%- endif -%};

{% endif %}

insert into {{ target }} ({{ dest_cols_csv }})
Expand Down

0 comments on commit 773e8d3

Please sign in to comment.