From 9369e26a44af2d51df701c71754bfcf310ddd63a Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Wed, 14 Jun 2023 14:06:37 +0200 Subject: [PATCH 1/6] send dependencies.json in client-config request --- buildpack/databroker/business_events.py | 29 ++++++++++++++----- tests/unit/test_databroker_business_events.py | 8 +++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index e8f7a39b5..28807f47f 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -3,11 +3,15 @@ Extract Business Events configuration from vcap services and create mx constants """ +import os import logging import requests +import json from buildpack import util +BASE_PATH = os.getcwd() + CONSTANTS_PREFIX = "BusinessEvents" BE_CONSTANTS_TO_INJECT = [ @@ -30,16 +34,16 @@ def update_config(m2ee, vcap_services_data): logging.debug("Business Events config added to MicroflowConstants") -def _get_client_config(url, auth_token, version): +def _put_client_config(url, auth_token, dependencies_json): headers = { - "Authorization": f"Bearer {auth_token}", - "X-Version": version, + "Authorization": f"Bearer {auth_token}" } - resp = requests.get( + resp = requests.put( url=url, headers=headers, - timeout=30, + json=dependencies_json, + timeout=30 ) resp.raise_for_status() return resp.text @@ -55,6 +59,17 @@ def _configure_business_events_metrics(be_config, existing_constants): be_config[f"{CONSTANTS_PREFIX}.EnableHeartbeat"] = "true" +def _read_dependencies_json(): + file_path = os.path.join(BASE_PATH, "model", "dependencies.json") + try: + with open(file_path) as f: + return json.load(f) + except FileNotFoundError: + logging.error( + "Business Events: dependencies.json not found %s", file_path + ) + raise + def _get_config(vcap_services, existing_constants): be_config = {} try: @@ -76,10 +91,10 @@ def _get_config(vcap_services, existing_constants): be_config[ f"{CONSTANTS_PREFIX}.{constant}" ] = kafka_creds.get(constant, "") - client_config = _get_client_config( + client_config = _put_client_config( kafka_creds.get(CLIENT_CONFIG_URL_KEY, ""), auth_token, - "1", + _read_dependencies_json(), ) be_config[ f"{CONSTANTS_PREFIX}.ClientConfiguration" diff --git a/tests/unit/test_databroker_business_events.py b/tests/unit/test_databroker_business_events.py index 02fa2765e..473a35553 100644 --- a/tests/unit/test_databroker_business_events.py +++ b/tests/unit/test_databroker_business_events.py @@ -166,13 +166,17 @@ class TestDataBrokerBusinessEvents(unittest.TestCase): def _verify_vcap_info(self, is_apply_limits_present=True): with mock.patch( - "buildpack.databroker.business_events._get_client_config", + "buildpack.databroker.business_events._put_client_config", mock.MagicMock(return_value=self.expected_client_config), - ): + ), mock.patch( + "buildpack.databroker.business_events._read_dependencies_json", + mock.MagicMock(return_value={})) as mock_read_dependencies_json: business_events_cfg = business_events._get_config( util.get_vcap_services_data(), self.module_constants_with_metrics, ) + mock_read_dependencies_json.assert_called_once() + prefix = business_events.CONSTANTS_PREFIX assert business_events_cfg[f"{prefix}.ServerUrl"] == self.server_url From 2a3a1acacaf3ebda7b0366739d2abac8e2331494 Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Wed, 21 Jun 2023 17:31:11 +0200 Subject: [PATCH 2/6] slight change in api call request to databroker client-config --- buildpack/databroker/business_events.py | 17 ++++++++++------- tests/unit/test_databroker_business_events.py | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index 28807f47f..922c909da 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -6,7 +6,6 @@ import os import logging import requests -import json from buildpack import util @@ -34,15 +33,18 @@ def update_config(m2ee, vcap_services_data): logging.debug("Business Events config added to MicroflowConstants") -def _put_client_config(url, auth_token, dependencies_json): +def _put_client_config(url, auth_token, version, dependencies_json_str): headers = { - "Authorization": f"Bearer {auth_token}" + "Authorization": f"Bearer {auth_token}", + "X-Version": version, } resp = requests.put( url=url, headers=headers, - json=dependencies_json, + json={ + "dependencies": dependencies_json_str + }, timeout=30 ) resp.raise_for_status() @@ -59,11 +61,11 @@ def _configure_business_events_metrics(be_config, existing_constants): be_config[f"{CONSTANTS_PREFIX}.EnableHeartbeat"] = "true" -def _read_dependencies_json(): +def _read_dependencies_json_as_str(): file_path = os.path.join(BASE_PATH, "model", "dependencies.json") try: with open(file_path) as f: - return json.load(f) + return f.read() except FileNotFoundError: logging.error( "Business Events: dependencies.json not found %s", file_path @@ -94,7 +96,8 @@ def _get_config(vcap_services, existing_constants): client_config = _put_client_config( kafka_creds.get(CLIENT_CONFIG_URL_KEY, ""), auth_token, - _read_dependencies_json(), + "2", + _read_dependencies_json_as_str(), ) be_config[ f"{CONSTANTS_PREFIX}.ClientConfiguration" diff --git a/tests/unit/test_databroker_business_events.py b/tests/unit/test_databroker_business_events.py index 473a35553..d11d15681 100644 --- a/tests/unit/test_databroker_business_events.py +++ b/tests/unit/test_databroker_business_events.py @@ -169,13 +169,13 @@ def _verify_vcap_info(self, is_apply_limits_present=True): "buildpack.databroker.business_events._put_client_config", mock.MagicMock(return_value=self.expected_client_config), ), mock.patch( - "buildpack.databroker.business_events._read_dependencies_json", - mock.MagicMock(return_value={})) as mock_read_dependencies_json: + "buildpack.databroker.business_events._read_dependencies_json_as_str", + mock.MagicMock(return_value="")) as mock_read_dependencies_json_as_str: business_events_cfg = business_events._get_config( util.get_vcap_services_data(), self.module_constants_with_metrics, ) - mock_read_dependencies_json.assert_called_once() + mock_read_dependencies_json_as_str.assert_called_once() prefix = business_events.CONSTANTS_PREFIX From 68df30766a548d9833a7d8aece2a278f6b93c57a Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Thu, 22 Jun 2023 11:24:40 +0200 Subject: [PATCH 3/6] add retries to clientconfig call --- buildpack/databroker/business_events.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index 922c909da..72ac7838d 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -6,6 +6,7 @@ import os import logging import requests +from requests.adapters import HTTPAdapter, Retry from buildpack import util @@ -39,7 +40,15 @@ def _put_client_config(url, auth_token, version, dependencies_json_str): "X-Version": version, } - resp = requests.put( + session = requests.Session() + retries = Retry(total=2, + backoff_factor=0.1, + status_forcelist=[ 500, 502, 503, 504 ]) + adapter = HTTPAdapter(max_retries=retries) + session.mount('http://', adapter) + session.mount('https://', adapter) + + resp = session.put( url=url, headers=headers, json={ From 3e5690aaed1cd3534eb5cc7ebcfddbb63b4d0e74 Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Mon, 26 Jun 2023 09:56:34 +0200 Subject: [PATCH 4/6] change version 2 to version 1 --- buildpack/databroker/business_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index 72ac7838d..73e6bbbf9 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -105,7 +105,7 @@ def _get_config(vcap_services, existing_constants): client_config = _put_client_config( kafka_creds.get(CLIENT_CONFIG_URL_KEY, ""), auth_token, - "2", + "1", _read_dependencies_json_as_str(), ) be_config[ From d390cd4337e006faf7f4e01b8ab1a0718c3545d2 Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Mon, 26 Jun 2023 16:27:50 +0200 Subject: [PATCH 5/6] remove redundant http filter for request creation --- buildpack/databroker/business_events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index 73e6bbbf9..33d69da67 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -45,7 +45,6 @@ def _put_client_config(url, auth_token, version, dependencies_json_str): backoff_factor=0.1, status_forcelist=[ 500, 502, 503, 504 ]) adapter = HTTPAdapter(max_retries=retries) - session.mount('http://', adapter) session.mount('https://', adapter) resp = session.put( From cb78b7299587b14221fbebe41151ce2be1ea9248 Mon Sep 17 00:00:00 2001 From: Albert de Vries Date: Tue, 27 Jun 2023 10:51:08 +0200 Subject: [PATCH 6/6] formatting --- buildpack/databroker/business_events.py | 17 ++++++----------- tests/unit/test_databroker_business_events.py | 3 ++- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/buildpack/databroker/business_events.py b/buildpack/databroker/business_events.py index 33d69da67..67c8342e9 100644 --- a/buildpack/databroker/business_events.py +++ b/buildpack/databroker/business_events.py @@ -41,19 +41,15 @@ def _put_client_config(url, auth_token, version, dependencies_json_str): } session = requests.Session() - retries = Retry(total=2, - backoff_factor=0.1, - status_forcelist=[ 500, 502, 503, 504 ]) + retries = Retry(total=2, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) adapter = HTTPAdapter(max_retries=retries) - session.mount('https://', adapter) + session.mount("https://", adapter) resp = session.put( url=url, headers=headers, - json={ - "dependencies": dependencies_json_str - }, - timeout=30 + json={"dependencies": dependencies_json_str}, + timeout=30, ) resp.raise_for_status() return resp.text @@ -75,11 +71,10 @@ def _read_dependencies_json_as_str(): with open(file_path) as f: return f.read() except FileNotFoundError: - logging.error( - "Business Events: dependencies.json not found %s", file_path - ) + logging.error("Business Events: dependencies.json not found %s", file_path) raise + def _get_config(vcap_services, existing_constants): be_config = {} try: diff --git a/tests/unit/test_databroker_business_events.py b/tests/unit/test_databroker_business_events.py index d11d15681..d3c1e6e23 100644 --- a/tests/unit/test_databroker_business_events.py +++ b/tests/unit/test_databroker_business_events.py @@ -170,7 +170,8 @@ def _verify_vcap_info(self, is_apply_limits_present=True): mock.MagicMock(return_value=self.expected_client_config), ), mock.patch( "buildpack.databroker.business_events._read_dependencies_json_as_str", - mock.MagicMock(return_value="")) as mock_read_dependencies_json_as_str: + mock.MagicMock(return_value=""), + ) as mock_read_dependencies_json_as_str: business_events_cfg = business_events._get_config( util.get_vcap_services_data(), self.module_constants_with_metrics,