Skip to content

Commit

Permalink
Use insert overwrite for the first batch of seed (#149)
Browse files Browse the repository at this point in the history
resolves #114

### Description

Uses `insert overwrite` for the first batch of seed to help seeds in external tables.

This must be a temporary fix and should follow dbt-labs/dbt-spark#182.
  • Loading branch information
ueshin authored Aug 25, 2022
1 parent 0ca1bce commit 44ec7d9
Showing 1 changed file with 38 additions and 0 deletions.
38 changes: 38 additions & 0 deletions dbt/include/databricks/macros/materializations/seed.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,44 @@
{{ return('%s') }}
{% endmacro %}

{% macro databricks__load_csv_rows(model, agate_table) %}

{% set batch_size = get_batch_size() %}
{% set column_override = model['config'].get('column_types', {}) %}

{% set statements = [] %}

{% for chunk in agate_table.rows | batch(batch_size) %}
{% set bindings = [] %}

{% for row in chunk %}
{% do bindings.extend(row) %}
{% endfor %}

{% set sql %}
insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values
{% for row in chunk -%}
({%- for col_name in agate_table.column_names -%}
{%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
{%- set type = column_override.get(col_name, inferred_type) -%}
cast({{ get_binding_char() }} as {{type}})
{%- if not loop.last%},{%- endif %}
{%- endfor -%})
{%- if not loop.last%},{%- endif %}
{%- endfor %}
{% endset %}

{% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}

{% if loop.index0 == 0 %}
{% do statements.append(sql) %}
{% endif %}
{% endfor %}

{# Return SQL so we can render it out into the compiled files #}
{{ return(statements[0]) }}
{% endmacro %}

{% macro databricks__create_csv_table(model, agate_table) %}
{%- set column_override = model['config'].get('column_types', {}) -%}
{%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
Expand Down

0 comments on commit 44ec7d9

Please sign in to comment.