From 58f3a71c892cc851413536347ee6b1acc54a16d3 Mon Sep 17 00:00:00 2001 From: kaareseras Date: Thu, 6 Jun 2024 12:25:18 +0000 Subject: [PATCH 1/7] chore: Refactor Azure Data Explorer client initialization Refactor the initialization of the Azure Data Explorer client to separate the creation of the ingest and query clients. Thisremoves the bug where the wrong URL was used when either testing for connectivity or inserting data --- .../components/azure_data_explorer/client.py | 43 +++++++++++++------ .../azure_data_explorer/config_flow.py | 4 +- .../components/azure_data_explorer/const.py | 2 +- .../azure_data_explorer/strings.json | 19 +++++--- tests/components/azure_data_explorer/const.py | 8 ++-- 5 files changed, 50 insertions(+), 26 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/client.py b/homeassistant/components/azure_data_explorer/client.py index 40528bc6a6ffe..f6803be90373b 100644 --- a/homeassistant/components/azure_data_explorer/client.py +++ b/homeassistant/components/azure_data_explorer/client.py @@ -23,7 +23,7 @@ CONF_APP_REG_ID, CONF_APP_REG_SECRET, CONF_AUTHORITY_ID, - CONF_USE_FREE, + CONF_USE_QUEUED_CLIENT, ) _LOGGER = logging.getLogger(__name__) @@ -35,7 +35,14 @@ class AzureDataExplorerClient: def __init__(self, data: Mapping[str, Any]) -> None: """Create the right class.""" + # User provided data with the Ingest- in the URI self._cluster_ingest_uri = data[CONF_ADX_CLUSTER_INGEST_URI] + + # Remove the Ingest- from the URI + self._cluster_query_uri = data[CONF_ADX_CLUSTER_INGEST_URI].replace( + "ingest-", "" + ) + self._database = data[CONF_ADX_DATABASE_NAME] self._table = data[CONF_ADX_TABLE_NAME] self._ingestion_properties = IngestionProperties( @@ -45,24 +52,36 @@ def __init__(self, data: Mapping[str, Any]) -> None: ingestion_mapping_reference="ha_json_mapping", ) - # Create cLient for ingesting and querying data - kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( - self._cluster_ingest_uri, - data[CONF_APP_REG_ID], - data[CONF_APP_REG_SECRET], - data[CONF_AUTHORITY_ID], + # Create cLient for ingesting data + kcsb_ingest = ( + KustoConnectionStringBuilder.with_aad_application_key_authentication( + self._cluster_ingest_uri, + data[CONF_APP_REG_ID], + data[CONF_APP_REG_SECRET], + data[CONF_AUTHORITY_ID], + ) + ) + + # Create cLient for reading data + kcsb_queri = ( + KustoConnectionStringBuilder.with_aad_application_key_authentication( + self._cluster_query_uri, + data[CONF_APP_REG_ID], + data[CONF_APP_REG_SECRET], + data[CONF_AUTHORITY_ID], + ) ) - if data[CONF_USE_FREE] is True: + if data[CONF_USE_QUEUED_CLIENT] is True: # Queded is the only option supported on free tear of ADX - self.write_client = QueuedIngestClient(kcsb) + self.write_client = QueuedIngestClient(kcsb_ingest) else: - self.write_client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb) + self.write_client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb_ingest) - self.query_client = KustoClient(kcsb) + self.query_client = KustoClient(kcsb_queri) def test_connection(self) -> None: - """Test connection, will throw Exception when it cannot connect.""" + """Test connection, will throw Exception if it cannot connect.""" query = f"{self._table} | take 1" diff --git a/homeassistant/components/azure_data_explorer/config_flow.py b/homeassistant/components/azure_data_explorer/config_flow.py index d8390246b4199..5ad1ed8689d5a 100644 --- a/homeassistant/components/azure_data_explorer/config_flow.py +++ b/homeassistant/components/azure_data_explorer/config_flow.py @@ -19,7 +19,7 @@ CONF_APP_REG_ID, CONF_APP_REG_SECRET, CONF_AUTHORITY_ID, - CONF_USE_FREE, + CONF_USE_QUEUED_CLIENT, DEFAULT_OPTIONS, DOMAIN, ) @@ -34,7 +34,7 @@ vol.Required(CONF_APP_REG_ID): str, vol.Required(CONF_APP_REG_SECRET): str, vol.Required(CONF_AUTHORITY_ID): str, - vol.Optional(CONF_USE_FREE, default=False): bool, + vol.Optional(CONF_USE_QUEUED_CLIENT, default=False): bool, } ) diff --git a/homeassistant/components/azure_data_explorer/const.py b/homeassistant/components/azure_data_explorer/const.py index ca98110597a91..a88a6b8b94fb0 100644 --- a/homeassistant/components/azure_data_explorer/const.py +++ b/homeassistant/components/azure_data_explorer/const.py @@ -17,7 +17,7 @@ CONF_SEND_INTERVAL = "send_interval" CONF_MAX_DELAY = "max_delay" CONF_FILTER = DATA_FILTER = "filter" -CONF_USE_FREE = "use_queued_ingestion" +CONF_USE_QUEUED_CLIENT = "use_queued_ingestion" DATA_HUB = "hub" STEP_USER = "user" diff --git a/homeassistant/components/azure_data_explorer/strings.json b/homeassistant/components/azure_data_explorer/strings.json index a3a82a6eb3c65..2f903786a6e9f 100644 --- a/homeassistant/components/azure_data_explorer/strings.json +++ b/homeassistant/components/azure_data_explorer/strings.json @@ -2,16 +2,21 @@ "config": { "step": { "user": { - "title": "Setup your Azure Data Explorer integration", - "description": "Enter connection details.", "data": { - "clusteringesturi": "Cluster Ingest URI", - "database": "Database name", - "table": "Table name", + "cluster_ingest_uri": "Cluster Ingest URI ", + "authority_id": "Authority ID", "client_id": "Client ID", "client_secret": "Client secret", - "authority_id": "Authority ID" - } + "database": "Database name", + "table": "Table name", + "use_queued_ingestion": "Use Queued ingestion (not for free cluster)" + }, + "data_description": { + "cluster_ingest_uri": "Ingest URI of the cluster, user the 'ingest-' URI", + "use_queued_ingestion": "Uncheck if using free cluster" + }, + "description": "Enter connection details.", + "title": "Setup your Azure Data Explorer integration" } }, "error": { diff --git a/tests/components/azure_data_explorer/const.py b/tests/components/azure_data_explorer/const.py index d29f4d5ba93db..d20be1584a198 100644 --- a/tests/components/azure_data_explorer/const.py +++ b/tests/components/azure_data_explorer/const.py @@ -8,7 +8,7 @@ CONF_APP_REG_SECRET, CONF_AUTHORITY_ID, CONF_SEND_INTERVAL, - CONF_USE_FREE, + CONF_USE_QUEUED_CLIENT, ) AZURE_DATA_EXPLORER_PATH = "homeassistant.components.azure_data_explorer" @@ -29,7 +29,7 @@ } BASIC_OPTIONS = { - CONF_USE_FREE: False, + CONF_USE_QUEUED_CLIENT: False, CONF_SEND_INTERVAL: 5, } @@ -39,10 +39,10 @@ BASE_CONFIG_IMPORT = { CONF_ADX_CLUSTER_INGEST_URI: "https://cluster.region.kusto.windows.net", - CONF_USE_FREE: False, + CONF_USE_QUEUED_CLIENT: False, CONF_SEND_INTERVAL: 5, } -FREE_OPTIONS = {CONF_USE_FREE: True, CONF_SEND_INTERVAL: 5} +FREE_OPTIONS = {CONF_USE_QUEUED_CLIENT: True, CONF_SEND_INTERVAL: 5} BASE_CONFIG_FREE = BASE_CONFIG | FREE_OPTIONS From 59902404737f3fcb1a9de3e6642838ff4d35cc59 Mon Sep 17 00:00:00 2001 From: kaareseras Date: Fri, 7 Jun 2024 14:37:17 +0000 Subject: [PATCH 2/7] Refactor Azure Data Explorer client initialization --- homeassistant/components/azure_data_explorer/client.py | 6 +++--- homeassistant/components/azure_data_explorer/strings.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/client.py b/homeassistant/components/azure_data_explorer/client.py index f6803be90373b..a0fc37dfa4466 100644 --- a/homeassistant/components/azure_data_explorer/client.py +++ b/homeassistant/components/azure_data_explorer/client.py @@ -62,8 +62,8 @@ def __init__(self, data: Mapping[str, Any]) -> None: ) ) - # Create cLient for reading data - kcsb_queri = ( + # Create cLient for querying data + kcsb_query = ( KustoConnectionStringBuilder.with_aad_application_key_authentication( self._cluster_query_uri, data[CONF_APP_REG_ID], @@ -78,7 +78,7 @@ def __init__(self, data: Mapping[str, Any]) -> None: else: self.write_client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb_ingest) - self.query_client = KustoClient(kcsb_queri) + self.query_client = KustoClient(kcsb_query) def test_connection(self) -> None: """Test connection, will throw Exception if it cannot connect.""" diff --git a/homeassistant/components/azure_data_explorer/strings.json b/homeassistant/components/azure_data_explorer/strings.json index 2f903786a6e9f..dd95479427ab9 100644 --- a/homeassistant/components/azure_data_explorer/strings.json +++ b/homeassistant/components/azure_data_explorer/strings.json @@ -9,11 +9,11 @@ "client_secret": "Client secret", "database": "Database name", "table": "Table name", - "use_queued_ingestion": "Use Queued ingestion (not for free cluster)" + "use_queued_ingestion": "Use Queued ingestion (Mandotory for free cluster)" }, "data_description": { - "cluster_ingest_uri": "Ingest URI of the cluster, user the 'ingest-' URI", - "use_queued_ingestion": "Uncheck if using free cluster" + "cluster_ingest_uri": "Ingest-URI of the cluster, use the 'https://ingest-xxxx.xx.xx' URI", + "use_queued_ingestion": "Check if using free cluster" }, "description": "Enter connection details.", "title": "Setup your Azure Data Explorer integration" From f55cf3687d6e4f01abd1cce74d833b2caf899d1a Mon Sep 17 00:00:00 2001 From: kaareseras Date: Fri, 7 Jun 2024 19:12:44 +0000 Subject: [PATCH 3/7] fixed filter --- .../components/azure_data_explorer/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/__init__.py b/homeassistant/components/azure_data_explorer/__init__.py index 62718d6938e4e..93d5223b9a75f 100644 --- a/homeassistant/components/azure_data_explorer/__init__.py +++ b/homeassistant/components/azure_data_explorer/__init__.py @@ -62,13 +62,13 @@ async def async_setup(hass: HomeAssistant, yaml_config: ConfigType) -> bool: Adds an empty filter to hass data. Tries to get a filter from yaml, if present set to hass data. - If config is empty after getting the filter, return, otherwise emit - deprecated warning and pass the rest to the config flow. """ - hass.data.setdefault(DOMAIN, {DATA_FILTER: {}}) - if DOMAIN in yaml_config: - hass.data[DOMAIN][DATA_FILTER] = yaml_config[DOMAIN][CONF_FILTER] + hass.data.setdefault(DOMAIN, {DATA_FILTER: FILTER_SCHEMA({})}) + if DOMAIN not in yaml_config: + return True + hass.data[DOMAIN][DATA_FILTER] = yaml_config[DOMAIN].pop(CONF_FILTER) + return True From b9edda6f11a08b26ddd9fc02e57edba361dce584 Mon Sep 17 00:00:00 2001 From: kaareseras Date: Fri, 7 Jun 2024 19:44:08 +0000 Subject: [PATCH 4/7] fixed data bug --- homeassistant/components/azure_data_explorer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/azure_data_explorer/__init__.py b/homeassistant/components/azure_data_explorer/__init__.py index 93d5223b9a75f..641e397af1c26 100644 --- a/homeassistant/components/azure_data_explorer/__init__.py +++ b/homeassistant/components/azure_data_explorer/__init__.py @@ -207,6 +207,6 @@ def _parse_event( if "\n" in state.state: return None, dropped + 1 - json_event = str(json.dumps(obj=state, cls=JSONEncoder).encode("utf-8")) + json_event = json.dumps(obj=state, cls=JSONEncoder) return (json_event, dropped) From af7378cb1f9702e1b95a3a413c54cba869782f5d Mon Sep 17 00:00:00 2001 From: kaareseras Date: Sat, 8 Jun 2024 09:07:31 +0000 Subject: [PATCH 5/7] chore: Fix typos and improve consistency in Azure Data Explorer client code --- .../components/azure_data_explorer/client.py | 4 ++-- .../components/azure_data_explorer/strings.json | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/client.py b/homeassistant/components/azure_data_explorer/client.py index a0fc37dfa4466..946fc279d19f8 100644 --- a/homeassistant/components/azure_data_explorer/client.py +++ b/homeassistant/components/azure_data_explorer/client.py @@ -52,7 +52,7 @@ def __init__(self, data: Mapping[str, Any]) -> None: ingestion_mapping_reference="ha_json_mapping", ) - # Create cLient for ingesting data + # Create client for ingesting data kcsb_ingest = ( KustoConnectionStringBuilder.with_aad_application_key_authentication( self._cluster_ingest_uri, @@ -62,7 +62,7 @@ def __init__(self, data: Mapping[str, Any]) -> None: ) ) - # Create cLient for querying data + # Create client for querying data kcsb_query = ( KustoConnectionStringBuilder.with_aad_application_key_authentication( self._cluster_query_uri, diff --git a/homeassistant/components/azure_data_explorer/strings.json b/homeassistant/components/azure_data_explorer/strings.json index dd95479427ab9..a2203871a216c 100644 --- a/homeassistant/components/azure_data_explorer/strings.json +++ b/homeassistant/components/azure_data_explorer/strings.json @@ -2,21 +2,20 @@ "config": { "step": { "user": { + "title": "Setup your Azure Data Explorer integration", + "description": "Enter connection details.\n\n'Use queued ingestion' must be checked when using ADX free cluster.", "data": { - "cluster_ingest_uri": "Cluster Ingest URI ", + "cluster_ingest_uri": "Cluster Ingest URI", "authority_id": "Authority ID", "client_id": "Client ID", "client_secret": "Client secret", "database": "Database name", "table": "Table name", - "use_queued_ingestion": "Use Queued ingestion (Mandotory for free cluster)" + "use_queued_ingestion": "Use queued ingestion" }, "data_description": { - "cluster_ingest_uri": "Ingest-URI of the cluster, use the 'https://ingest-xxxx.xx.xx' URI", - "use_queued_ingestion": "Check if using free cluster" - }, - "description": "Enter connection details.", - "title": "Setup your Azure Data Explorer integration" + "cluster_ingest_uri": "Ingest-URI of the cluster" + } } }, "error": { From 53d194e27a4ac9630295c70957483bc6ddb72897 Mon Sep 17 00:00:00 2001 From: kaareseras Date: Mon, 10 Jun 2024 14:01:09 +0000 Subject: [PATCH 6/7] refactoring client --- .../components/azure_data_explorer/client.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/client.py b/homeassistant/components/azure_data_explorer/client.py index 946fc279d19f8..88609ff8e103c 100644 --- a/homeassistant/components/azure_data_explorer/client.py +++ b/homeassistant/components/azure_data_explorer/client.py @@ -35,14 +35,6 @@ class AzureDataExplorerClient: def __init__(self, data: Mapping[str, Any]) -> None: """Create the right class.""" - # User provided data with the Ingest- in the URI - self._cluster_ingest_uri = data[CONF_ADX_CLUSTER_INGEST_URI] - - # Remove the Ingest- from the URI - self._cluster_query_uri = data[CONF_ADX_CLUSTER_INGEST_URI].replace( - "ingest-", "" - ) - self._database = data[CONF_ADX_DATABASE_NAME] self._table = data[CONF_ADX_TABLE_NAME] self._ingestion_properties = IngestionProperties( @@ -55,7 +47,7 @@ def __init__(self, data: Mapping[str, Any]) -> None: # Create client for ingesting data kcsb_ingest = ( KustoConnectionStringBuilder.with_aad_application_key_authentication( - self._cluster_ingest_uri, + data[CONF_ADX_CLUSTER_INGEST_URI], data[CONF_APP_REG_ID], data[CONF_APP_REG_SECRET], data[CONF_AUTHORITY_ID], @@ -65,7 +57,7 @@ def __init__(self, data: Mapping[str, Any]) -> None: # Create client for querying data kcsb_query = ( KustoConnectionStringBuilder.with_aad_application_key_authentication( - self._cluster_query_uri, + data[CONF_ADX_CLUSTER_INGEST_URI].replace("ingest-", ""), data[CONF_APP_REG_ID], data[CONF_APP_REG_SECRET], data[CONF_AUTHORITY_ID], From f39c922fe6ed7668c2958602cf96d01a5b8c67cb Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Tue, 11 Jun 2024 07:12:28 +0000 Subject: [PATCH 7/7] Use Booleanselector --- homeassistant/components/azure_data_explorer/__init__.py | 5 ++--- homeassistant/components/azure_data_explorer/config_flow.py | 3 ++- homeassistant/components/azure_data_explorer/strings.json | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/azure_data_explorer/__init__.py b/homeassistant/components/azure_data_explorer/__init__.py index 641e397af1c26..319f7e4389be6 100644 --- a/homeassistant/components/azure_data_explorer/__init__.py +++ b/homeassistant/components/azure_data_explorer/__init__.py @@ -65,9 +65,8 @@ async def async_setup(hass: HomeAssistant, yaml_config: ConfigType) -> bool: """ hass.data.setdefault(DOMAIN, {DATA_FILTER: FILTER_SCHEMA({})}) - if DOMAIN not in yaml_config: - return True - hass.data[DOMAIN][DATA_FILTER] = yaml_config[DOMAIN].pop(CONF_FILTER) + if DOMAIN in yaml_config: + hass.data[DOMAIN][DATA_FILTER] = yaml_config[DOMAIN].pop(CONF_FILTER) return True diff --git a/homeassistant/components/azure_data_explorer/config_flow.py b/homeassistant/components/azure_data_explorer/config_flow.py index 5ad1ed8689d5a..4ffb5ea7cf7e7 100644 --- a/homeassistant/components/azure_data_explorer/config_flow.py +++ b/homeassistant/components/azure_data_explorer/config_flow.py @@ -10,6 +10,7 @@ from homeassistant import config_entries from homeassistant.config_entries import ConfigFlowResult +from homeassistant.helpers.selector import BooleanSelector from . import AzureDataExplorerClient from .const import ( @@ -34,7 +35,7 @@ vol.Required(CONF_APP_REG_ID): str, vol.Required(CONF_APP_REG_SECRET): str, vol.Required(CONF_AUTHORITY_ID): str, - vol.Optional(CONF_USE_QUEUED_CLIENT, default=False): bool, + vol.Required(CONF_USE_QUEUED_CLIENT, default=False): BooleanSelector(), } ) diff --git a/homeassistant/components/azure_data_explorer/strings.json b/homeassistant/components/azure_data_explorer/strings.json index a2203871a216c..c8ec158a84454 100644 --- a/homeassistant/components/azure_data_explorer/strings.json +++ b/homeassistant/components/azure_data_explorer/strings.json @@ -3,7 +3,7 @@ "step": { "user": { "title": "Setup your Azure Data Explorer integration", - "description": "Enter connection details.\n\n'Use queued ingestion' must be checked when using ADX free cluster.", + "description": "Enter connection details", "data": { "cluster_ingest_uri": "Cluster Ingest URI", "authority_id": "Authority ID", @@ -14,7 +14,8 @@ "use_queued_ingestion": "Use queued ingestion" }, "data_description": { - "cluster_ingest_uri": "Ingest-URI of the cluster" + "cluster_ingest_uri": "Ingest-URI of the cluster", + "use_queued_ingestion": "Must be enabled when using ADX free cluster" } } },