updated how packages and secrets are passed to udf

DataEngineersNZ · Nov 1, 2023 · 25edafd · 25edafd
1 parent c3f2df5
commit 25edafd
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 31 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,14 @@
 
 * Modified Monitorial Alert Object with updated fields
 
+## 0.2.7
+
+* Monitorial: Email Alert Updates
+* Added in ability to set the timeout seconds for the task (timeout)
+* Added in ability to set the suspend task after number of failures (suspend_after_number_of_failures)
+* Added in the ability to add external_access_integrations to user defined functions
+* Added in the ability to add secrets to user defined functions
+
 ## 0.2.6
 
 * Modified alert materialisation to be just the Snowflake Alert Object

diff --git a/README.md b/README.md
@@ -436,27 +436,27 @@ To create a user defined function using Python, you need to add the following co
     is_secure= false,
     immutable=false,
     runtime_version = '3.8',
-    packages = "('numpy','pandas','xgboost==1.5.0')",
-    external_access_integrations = "(your_access_integration)",
-    secrets = "('cred' = oauth_token )"
+    packages = ['numpy','pandas','xgboost==1.5.0'],
+    external_access_integrations = "your_access_integration",
+    secrets = ["\'cred\' = oauth_token"]
     handler_name = 'udf',
     return_type = 'variant')
 }}
 ```
 
-| property                       | description                                                      | required | default                 |
-| ------------------------------ | ---------------------------------------------------------------- | -------- | ----------------------- |
-| `materialized`                 | specifies the type of materialisation to run                     | yes      | `user_defined_function` |
-| `preferred_language`           | specifies the landuage for the UDF function                      | yes      | `python`                |
-| `is_secure`                    | specifies the function whether it is secure or not?              | no       | `false`                 |
-| `immutable`                    | specifies the function is mutable or immutable                   | no       | `false`                 |
-| `return_type`                  | specifies the datatype for the return value                      | yes      |                         |
-| `parameters`                   | specifies the parameter for the function                         | no       |                         |
-| `runtime_version`              | specifies the version of python                                  | yes      |                         |
-| `packages`                     | specifies the packages required for the python function          | yes      |                         |
-| `handler_name`                 | specifies the handler name for the function                      | yes      |                         |
-| `external_access_integrations` | specifies the name of the external access integration to be used | no       |                         |
-| `secrets`                      | specifies the secrets that are to be used by the function        | no       |                         |
+| property                       | description                                                       | required | default                 |
+| ------------------------------ | ----------------------------------------------------------------- | -------- | ----------------------- |
+| `materialized`                 | specifies the type of materialisation to run                      | yes      | `user_defined_function` |
+| `preferred_language`           | specifies the landuage for the UDF function                       | yes      | `python`                |
+| `is_secure`                    | specifies the function whether it is secure or not?               | no       | `false`                 |
+| `immutable`                    | specifies the function is mutable or immutable                    | no       | `false`                 |
+| `return_type`                  | specifies the datatype for the return value                       | yes      |                         |
+| `parameters`                   | specifies the parameter for the function                          | no       |                         |
+| `runtime_version`              | specifies the version of python                                   | yes      |                         |
+| `packages`                     | specifies an array of packages required for the python function   | yes      |                         |
+| `handler_name`                 | specifies the handler name for the function                       | yes      |                         |
+| `external_access_integrations` | specifies the name of the external access integration to be used  | no       |                         |
+| `secrets`                      | specifies an array of secrets that are to be used by the function | no       |                         |
 
 ## Materialized View
 To create a Materialized View, you need to add the following config to the top of your model:

diff --git a/macros/user_defined_functions/snowflake_create_user_defined_functions_statement.sql b/macros/user_defined_functions/snowflake_create_user_defined_functions_statement.sql
@@ -6,29 +6,31 @@ CREATE OR REPLACE SECURE FUNCTION {{ relation.include(database=(not temporary),
 CREATE OR REPLACE FUNCTION {{ relation.include(database=(not temporary), schema=(not temporary)) }}({{ parameters }})
 {% endif %}
 RETURNS {{ return_type }}
-{% if preferred_language != 'sql' %}
+{% if preferred_language != 'sql' -%}
 LANGUAGE  {{ preferred_language }}
-{% endif %}
+{%- endif %}
 {% if immutable %}
-    IMMUTABLE
+IMMUTABLE
 {% else %}
-   VOLATILE
-{% endif %}
+VOLATILE
+{%- endif -%}
 
-{% if preferred_language == 'python'  %}
+{% if preferred_language == 'python' %}
 RUNTIME_VERSION = '{{ runtime_version }}'
 HANDLER = '{{ handler_name }}'
-PACKAGES = {{ packages }}
-{% if external_access_integrations is not none %}
-EXTERNAL_ACCESS_INTEGRATIONS = {{ external_access_integrations }}
-{% endif %}
-{% if secrets is not none %}
-SECRETS = {{ secrets }}
-{% endif %}
-{% elif preferred_language == 'java'  %}
+PACKAGES = ('{{ packages|join('\', \'') }}')
+{% if external_access_integrations is not none -%}
+EXTERNAL_ACCESS_INTEGRATIONS = ({{ external_access_integrations }})
+{%- endif %}
+{% if secrets is not none -%}
+SECRETS = ({{ secrets|join(', ') }})
+{%- endif %}
+{% if import_Path is not none -%}
+{%- endif %}
+{%- elif preferred_language == 'java' -%}
 handler = {{ handler_name }}
 target_path = {{ target_path }}
-{% endif %}
+{%- endif %}
 
 AS