fix: check for old tmp table before deleting it (#550)

Co-authored-by: Spencer horton <[email protected]>

Author: octiva

dbt/include/athena/macros/materializations/models/incremental/incremental.sql

@@ -1,5 +1,4 @@
 {% materialization incremental, adapter='athena' -%}
-
   {% set raw_strategy = config.get('incremental_strategy') or 'insert_overwrite' %}
   {% set table_type = config.get('table_type', default='hive') | lower %}
   {% set strategy = validate_get_incremental_strategy(raw_strategy, table_type) %}
@@ -11,7 +10,10 @@
   {% set force_batch = config.get('force_batch', False) | as_bool -%}
   {% set target_relation = this.incorporate(type='table') %}
   {% set existing_relation = load_relation(this) %}
-  {% set tmp_relation = make_temp_relation(this) %}
+  {% set old_tmp_relation = adapter.get_relation(identifier=target_relation.identifier ~ '__dbt_tmp',
+                                             schema=schema,
+                                             database=database) %}
+  {% set tmp_relation = make_temp_relation(target_relation, '__dbt_tmp') %}
 
   -- If no partitions are used with insert_overwrite, we fall back to append mode.
   {% if partitioned_by is none and strategy == 'insert_overwrite' %}
@@ -32,9 +34,8 @@
     {% set query_result = safe_create_table_as(False, target_relation, sql, force_batch) -%}
     {% set build_sql = "select '" ~ query_result ~ "'" -%}
   {% elif partitioned_by is not none and strategy == 'insert_overwrite' %}
-    {% set tmp_relation = make_temp_relation(target_relation) %}
-    {% if tmp_relation is not none %}
-      {% do drop_relation(tmp_relation) %}
+    {% if old_tmp_relation is not none %}
+      {% do drop_relation(old_tmp_relation) %}
     {% endif %}
     {% set query_result = safe_create_table_as(True, tmp_relation, sql, force_batch) -%}
     {% do delete_overlapping_partitions(target_relation, tmp_relation, partitioned_by) %}
@@ -44,9 +45,8 @@
     %}
     {% do to_drop.append(tmp_relation) %}
   {% elif strategy == 'append' %}
-    {% set tmp_relation = make_temp_relation(target_relation) %}
-    {% if tmp_relation is not none %}
-      {% do drop_relation(tmp_relation) %}
+    {% if old_tmp_relation is not none %}
+      {% do drop_relation(old_tmp_relation) %}
     {% endif %}
     {% set query_result = safe_create_table_as(True, tmp_relation, sql, force_batch) -%}
     {% set build_sql = incremental_insert(
@@ -74,9 +74,8 @@
         {% do exceptions.raise_compiler_error(inc_predicates_not_list) %}
       {% endif %}
     {% endif %}
-    {% set tmp_relation = make_temp_relation(target_relation) %}
-    {% if tmp_relation is not none %}
-      {% do drop_relation(tmp_relation) %}
+    {% if old_tmp_relation is not none %}
+      {% do drop_relation(old_tmp_relation) %}
     {% endif %}
     {% set query_result = safe_create_table_as(True, tmp_relation, sql, force_batch) -%}
     {% set build_sql = iceberg_merge(

dbt/include/athena/macros/materializations/models/table/table.sql

@@ -7,6 +7,9 @@
 
   {%- set table_type = config.get('table_type', default='hive') | lower -%}
   {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set old_tmp_relation = adapter.get_relation(identifier=identifier ~ '__ha',
+                                             schema=schema,
+                                             database=database) -%}
   {%- set is_ha = config.get('ha', default=false) -%}
   {%- set s3_data_dir = config.get('s3_data_dir', default=target.s3_data_dir) -%}
   {%- set s3_data_naming = config.get('s3_data_naming', default='table_unique') -%}
@@ -44,9 +47,9 @@
 
     -- for ha tables that are not in full refresh mode and when the relation exists we use the swap behavior
     {%- if is_ha and not is_full_refresh_mode and old_relation is not none -%}
-      -- drop the tmp_relation
-      {%- if tmp_relation is not none -%}
-        {%- do adapter.delete_from_glue_catalog(tmp_relation) -%}
+      -- drop the old_tmp_relation if it exists
+      {%- if old_tmp_relation is not none -%}
+        {%- do adapter.delete_from_glue_catalog(old_tmp_relation) -%}
       {%- endif -%}
 
       -- create tmp table
@@ -69,7 +72,6 @@
     {%- endif -%}
 
     {{ set_table_classification(target_relation) }}
-
   {%- else -%}
 
     {%- if old_relation is none -%}
@@ -80,9 +82,9 @@
         {%- do drop_relation(old_relation) -%}
         {%- do rename_relation(tmp_relation, target_relation) -%}
       {%- else -%}
-
-        {%- if tmp_relation is not none -%}
-          {%- do drop_relation(tmp_relation) -%}
+        -- delete old tmp iceberg table if it exists
+        {%- if old_tmp_relation is not none -%}
+          {%- do drop_relation(old_tmp_relation) -%}
         {%- endif -%}
 
         {%- set old_relation_bkp = make_temp_relation(old_relation, '__bkp') -%}