From 3a3699065411a1223575c9233c17690a9ccd377e Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Fri, 22 Nov 2024 14:19:02 -0800 Subject: [PATCH 01/15] rewrote get_config function to match jupyter server extension --- api/endpoints/constants.py | 3 +++ api/endpoints/environment.py | 26 +++++++++++++------------- api/settings.py | 2 -- 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 api/endpoints/constants.py diff --git a/api/endpoints/constants.py b/api/endpoints/constants.py new file mode 100644 index 0000000..91cae20 --- /dev/null +++ b/api/endpoints/constants.py @@ -0,0 +1,3 @@ +ADE_OPTIONS = ["ade.dit.maap-project.org", "ade.uat.maap-project.org", "ade.maap-project.org"] +DEFAULT_ADE = "ade.maap-project.org" +DEFAULT_API = "api.maap-project.org" \ No newline at end of file diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index dcb6563..e44d600 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -6,6 +6,8 @@ from api import settings import requests import urllib.parse +import os +import api.endpoints.constants as constants log = logging.getLogger(__name__) @@ -58,16 +60,14 @@ def get(self, ade_host): def get_config(ade_host): - req = requests.get(settings.MAAP_ENVIRONMENT_FILE) - if req.status_code == requests.codes.ok: - data = req.json() - else: - print('Content was not found.') - - base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) - - match = next((x for x in data if base_url in x['ade_server']), None) - maap_config = next((x for x in data if x['default_host'] == True), None) if match is None else match - - return maap_config - + api_host = os.getenv("MAAP_API_HOST", constants.DEFAULT_API) + print("graceal1 api host is ") + print(api_host) + maap_api_config_endpoint = os.getenv("MAAP_API_CONFIG_ENDPOINT", "api/environment/config") + print("graceal1 constants ADE options ") + print(constants.ADE_OPTIONS) + ade_host = ade_host if ade_host in constants.ADE_OPTIONS else os.getenv("MAAP_ADE_HOST", constants.DEFAULT_ADE) + environments_endpoint = "https://" + api_host + "/" + maap_api_config_endpoint + "/"+urllib.parse.quote(urllib.parse.quote("https://", safe=""))+ade_host + print("graceal1 environments endpoint is ") + print(environments_endpoint) + return requests.get(environments_endpoint).json() diff --git a/api/settings.py b/api/settings.py index 94dae86..708df75 100755 --- a/api/settings.py +++ b/api/settings.py @@ -43,8 +43,6 @@ def str2bool(v): GITLAB_TOKEN = os.getenv('GITLAB_TOKEN', 'foobar') GITLAB_API_TOKEN = os.getenv('GITLAB_API_TOKEN','') # New setting inherited from sister, remove comment after API is stable -MAAP_ENVIRONMENT_FILE = os.getenv('MAAP_ENVIRONMENT_FILE', 'https://raw.githubusercontent.com/MAAP-Project/maap-jupyter-ide/develop/maap_environments.json') - REPO_NAME = os.getenv('REPO_NAME', 'register-job') REPO_PATH = os.getenv('REPO_PATH', '/home/ubuntu/repo') VERSION = os.getenv('VERSION', 'master') From 3726a70ca0abac76605970798ac9df846133beed Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 26 Nov 2024 13:50:20 -0800 Subject: [PATCH 02/15] added additional print statements --- api/endpoints/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index e44d600..f30c233 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -62,7 +62,9 @@ def get(self, ade_host): def get_config(ade_host): api_host = os.getenv("MAAP_API_HOST", constants.DEFAULT_API) print("graceal1 api host is ") + print(os.getenv("MAAP_API_HOST")) print(api_host) + print(ade_host) maap_api_config_endpoint = os.getenv("MAAP_API_CONFIG_ENDPOINT", "api/environment/config") print("graceal1 constants ADE options ") print(constants.ADE_OPTIONS) From 5d822b8f53cb978e0d194168724e230756a74887 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 26 Nov 2024 13:57:50 -0800 Subject: [PATCH 03/15] added more print statements --- api/endpoints/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index f30c233..2a9d364 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) ns = api.namespace('environment', description='Operations related to the MAAP environment') - +print("graceal1 in environment.py for API") @ns.route('/config') class Config(Resource): @@ -60,6 +60,7 @@ def get(self, ade_host): def get_config(ade_host): + print("graceal1 in get_config for API ") api_host = os.getenv("MAAP_API_HOST", constants.DEFAULT_API) print("graceal1 api host is ") print(os.getenv("MAAP_API_HOST")) From 0aa26be062b777d33b7652e37e9b9e28566a20ef Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Mon, 2 Dec 2024 13:18:10 -0800 Subject: [PATCH 04/15] printing all environment variables --- api/endpoints/environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index 2a9d364..999b0d8 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -61,6 +61,7 @@ def get(self, ade_host): def get_config(ade_host): print("graceal1 in get_config for API ") + print(os.environ) api_host = os.getenv("MAAP_API_HOST", constants.DEFAULT_API) print("graceal1 api host is ") print(os.getenv("MAAP_API_HOST")) From 8f28db4048d03f0a1ee7191df92da27124e91713 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 16:12:12 -0800 Subject: [PATCH 05/15] reverted to the correct function and added back environment to settings.py --- api/endpoints/environment.py | 29 +++++++++++++++-------------- api/settings.py | 2 ++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index 999b0d8..42f9445 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -60,18 +60,19 @@ def get(self, ade_host): def get_config(ade_host): - print("graceal1 in get_config for API ") - print(os.environ) - api_host = os.getenv("MAAP_API_HOST", constants.DEFAULT_API) - print("graceal1 api host is ") - print(os.getenv("MAAP_API_HOST")) - print(api_host) + print("graceal1 in API get_config with") print(ade_host) - maap_api_config_endpoint = os.getenv("MAAP_API_CONFIG_ENDPOINT", "api/environment/config") - print("graceal1 constants ADE options ") - print(constants.ADE_OPTIONS) - ade_host = ade_host if ade_host in constants.ADE_OPTIONS else os.getenv("MAAP_ADE_HOST", constants.DEFAULT_ADE) - environments_endpoint = "https://" + api_host + "/" + maap_api_config_endpoint + "/"+urllib.parse.quote(urllib.parse.quote("https://", safe=""))+ade_host - print("graceal1 environments endpoint is ") - print(environments_endpoint) - return requests.get(environments_endpoint).json() + print(settings.MAAP_ENVIRONMENT_FILE) + req = requests.get(settings.MAAP_ENVIRONMENT_FILE) + if req.status_code == requests.codes.ok: + data = req.json() + else: + print('Content was not found.') + + base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) + + match = next((x for x in data if base_url in x['ade_server']), None) + maap_config = next((x for x in data if x['default_host'] == True), None) if match is None else match + print("graceal1 the maap_config is") + print(maap_config) + return maap_config diff --git a/api/settings.py b/api/settings.py index 708df75..94dae86 100755 --- a/api/settings.py +++ b/api/settings.py @@ -43,6 +43,8 @@ def str2bool(v): GITLAB_TOKEN = os.getenv('GITLAB_TOKEN', 'foobar') GITLAB_API_TOKEN = os.getenv('GITLAB_API_TOKEN','') # New setting inherited from sister, remove comment after API is stable +MAAP_ENVIRONMENT_FILE = os.getenv('MAAP_ENVIRONMENT_FILE', 'https://raw.githubusercontent.com/MAAP-Project/maap-jupyter-ide/develop/maap_environments.json') + REPO_NAME = os.getenv('REPO_NAME', 'register-job') REPO_PATH = os.getenv('REPO_PATH', '/home/ubuntu/repo') VERSION = os.getenv('VERSION', 'master') From 51739b1729a07c2f5187e6e3356ef501db3a6c7e Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 16:12:40 -0800 Subject: [PATCH 06/15] added more print statements --- api/endpoints/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index 42f9445..a011f3a 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -70,7 +70,8 @@ def get_config(ade_host): print('Content was not found.') base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) - + print("graceal1 req returned was") + print(req) match = next((x for x in data if base_url in x['ade_server']), None) maap_config = next((x for x in data if x['default_host'] == True), None) if match is None else match print("graceal1 the maap_config is") From 2e0f12fed05b23b3d300a73e6491e5dacb4a1f1b Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 16:29:27 -0800 Subject: [PATCH 07/15] changed the backup environments.json filepath --- api/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/settings.py b/api/settings.py index 94dae86..bb197dc 100755 --- a/api/settings.py +++ b/api/settings.py @@ -43,7 +43,7 @@ def str2bool(v): GITLAB_TOKEN = os.getenv('GITLAB_TOKEN', 'foobar') GITLAB_API_TOKEN = os.getenv('GITLAB_API_TOKEN','') # New setting inherited from sister, remove comment after API is stable -MAAP_ENVIRONMENT_FILE = os.getenv('MAAP_ENVIRONMENT_FILE', 'https://raw.githubusercontent.com/MAAP-Project/maap-jupyter-ide/develop/maap_environments.json') +MAAP_ENVIRONMENT_FILE = os.getenv('MAAP_ENVIRONMENT_FILE', 'https://raw.githubusercontent.com/MAAP-Project/maap-workspaces/refs/heads/main/jupyterlab/environments.json') REPO_NAME = os.getenv('REPO_NAME', 'register-job') REPO_PATH = os.getenv('REPO_PATH', '/home/ubuntu/repo') From 154e28ed6a3183504554ffd7e235d74527d9f8ee Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 17:18:32 -0800 Subject: [PATCH 08/15] now reading from environments.json in MAAP API --- api/endpoints/constants.py | 3 --- api/endpoints/environment.py | 15 +++++---------- api/endpoints/environments.json | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 13 deletions(-) delete mode 100644 api/endpoints/constants.py create mode 100644 api/endpoints/environments.json diff --git a/api/endpoints/constants.py b/api/endpoints/constants.py deleted file mode 100644 index 91cae20..0000000 --- a/api/endpoints/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -ADE_OPTIONS = ["ade.dit.maap-project.org", "ade.uat.maap-project.org", "ade.maap-project.org"] -DEFAULT_ADE = "ade.maap-project.org" -DEFAULT_API = "api.maap-project.org" \ No newline at end of file diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index a011f3a..afab2e7 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -6,8 +6,7 @@ from api import settings import requests import urllib.parse -import os -import api.endpoints.constants as constants +import json log = logging.getLogger(__name__) @@ -62,16 +61,12 @@ def get(self, ade_host): def get_config(ade_host): print("graceal1 in API get_config with") print(ade_host) - print(settings.MAAP_ENVIRONMENT_FILE) - req = requests.get(settings.MAAP_ENVIRONMENT_FILE) - if req.status_code == requests.codes.ok: - data = req.json() - else: - print('Content was not found.') + with open("environments.json") as f: + data = json.loads(f) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) - print("graceal1 req returned was") - print(req) + print("graceal1 req returned is") + print(data) match = next((x for x in data if base_url in x['ade_server']), None) maap_config = next((x for x in data if x['default_host'] == True), None) if match is None else match print("graceal1 the maap_config is") diff --git a/api/endpoints/environments.json b/api/endpoints/environments.json new file mode 100644 index 0000000..f4047ed --- /dev/null +++ b/api/endpoints/environments.json @@ -0,0 +1,33 @@ +[ + { + "environment": "dit", + "ade_server": "ade.dit.maap-project.org", + "api_server": "api.dit.maap-project.org", + "auth_server": "auth.dit.maap-project.org", + "mas_server": "repo.dit.maap-project.org", + "edsc_server": "ade.dit.maap-project.org:30052", + "workspace_bucket": "maap-dit-workspace", + "kibana_url": "https://35.88.169.231/metrics/goto/4f1115df1b6563782292fb1f75676f95", + "default_host": true + }, + { + "environment": "uat", + "ade_server": "ade.uat.maap-project.org", + "api_server": "api.uat.maap-project.org", + "auth_server": "auth.uat.maap-project.org", + "mas_server": "repo.uat.maap-project.org", + "edsc_server": "ade.uat.maap-project.org:30052", + "workspace_bucket": "maap-uat-workspace", + "default_host": false + }, + { + "environment": "ops", + "ade_server": "ade.maap-project.org", + "api_server": "api.maap-project.org", + "auth_server": "auth.maap-project.org", + "mas_server": "mas.maap-project.org", + "edsc_server": "ade.maap-project.org:30052", + "workspace_bucket": "maap-ops-workspace", + "default_host": false + } +] From 860dc6a7789e7c10b9f633aedce52ec89c76c43f Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 17:30:49 -0800 Subject: [PATCH 09/15] updated import for environments.json --- api/endpoints/environment.py | 4 +++- api/settings.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index afab2e7..b27cfbd 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -7,6 +7,7 @@ import requests import urllib.parse import json +import os log = logging.getLogger(__name__) @@ -61,7 +62,8 @@ def get(self, ade_host): def get_config(ade_host): print("graceal1 in API get_config with") print(ade_host) - with open("environments.json") as f: + ROOT = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(ROOT, "environments.json")) as f: data = json.loads(f) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) diff --git a/api/settings.py b/api/settings.py index bb197dc..708df75 100755 --- a/api/settings.py +++ b/api/settings.py @@ -43,8 +43,6 @@ def str2bool(v): GITLAB_TOKEN = os.getenv('GITLAB_TOKEN', 'foobar') GITLAB_API_TOKEN = os.getenv('GITLAB_API_TOKEN','') # New setting inherited from sister, remove comment after API is stable -MAAP_ENVIRONMENT_FILE = os.getenv('MAAP_ENVIRONMENT_FILE', 'https://raw.githubusercontent.com/MAAP-Project/maap-workspaces/refs/heads/main/jupyterlab/environments.json') - REPO_NAME = os.getenv('REPO_NAME', 'register-job') REPO_PATH = os.getenv('REPO_PATH', '/home/ubuntu/repo') VERSION = os.getenv('VERSION', 'master') From baeca713885f301ea2b4f45ef6291c1e4dc05d70 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 3 Dec 2024 17:43:24 -0800 Subject: [PATCH 10/15] corrected to json.load --- api/endpoints/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index b27cfbd..d03bc99 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -64,7 +64,7 @@ def get_config(ade_host): print(ade_host) ROOT = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(ROOT, "environments.json")) as f: - data = json.loads(f) + data = json.load(f) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) print("graceal1 req returned is") From cd652537d04d2867034317f4933276dd3f61db02 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Dec 2024 10:48:26 -0800 Subject: [PATCH 11/15] removed print statements --- api/endpoints/environment.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index d03bc99..8083e3b 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -12,7 +12,6 @@ log = logging.getLogger(__name__) ns = api.namespace('environment', description='Operations related to the MAAP environment') -print("graceal1 in environment.py for API") @ns.route('/config') class Config(Resource): @@ -60,17 +59,12 @@ def get(self, ade_host): def get_config(ade_host): - print("graceal1 in API get_config with") - print(ade_host) ROOT = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(ROOT, "environments.json")) as f: data = json.load(f) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) - print("graceal1 req returned is") - print(data) + match = next((x for x in data if base_url in x['ade_server']), None) maap_config = next((x for x in data if x['default_host'] == True), None) if match is None else match - print("graceal1 the maap_config is") - print(maap_config) return maap_config From a0521bee5542761b05b9ad29f0dba252398e2d60 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Dec 2024 17:26:07 -0800 Subject: [PATCH 12/15] added error handling --- api/endpoints/environment.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index 8083e3b..e77d1fa 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -59,9 +59,14 @@ def get(self, ade_host): def get_config(ade_host): - ROOT = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(ROOT, "environments.json")) as f: - data = json.load(f) + try: + ROOT = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(ROOT, "environments.json")) as f: + data = json.load(f) + except FileNotFoundError: + msg = "environments.json file could not be found" + logging.exception(msg) + raise Exception(msg) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host))) From 5149745cee0c098acd26c5071971d10d7558d9b6 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Dec 2024 17:27:04 -0800 Subject: [PATCH 13/15] purposely added error --- api/endpoints/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index e77d1fa..912ad13 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -61,7 +61,7 @@ def get(self, ade_host): def get_config(ade_host): try: ROOT = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(ROOT, "environments.json")) as f: + with open(os.path.join(ROOT, "environments1.json")) as f: data = json.load(f) except FileNotFoundError: msg = "environments.json file could not be found" From 924b0c700b5c789577b835725119c02a703e5bcb Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Dec 2024 17:33:53 -0800 Subject: [PATCH 14/15] corrected mistake --- api/endpoints/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index 912ad13..e77d1fa 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -61,7 +61,7 @@ def get(self, ade_host): def get_config(ade_host): try: ROOT = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(ROOT, "environments1.json")) as f: + with open(os.path.join(ROOT, "environments.json")) as f: data = json.load(f) except FileNotFoundError: msg = "environments.json file could not be found" From 390bc7f52d6807273b8b59ec5ee2dd46598a2684 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Thu, 5 Dec 2024 09:47:44 -0800 Subject: [PATCH 15/15] changed to FileNotFoundError to be more descriptive to pass test --- api/endpoints/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/endpoints/environment.py b/api/endpoints/environment.py index e77d1fa..5298a54 100755 --- a/api/endpoints/environment.py +++ b/api/endpoints/environment.py @@ -66,7 +66,7 @@ def get_config(ade_host): except FileNotFoundError: msg = "environments.json file could not be found" logging.exception(msg) - raise Exception(msg) + raise FileNotFoundError(msg) base_url = "{0.netloc}".format(urlsplit(urllib.parse.unquote(ade_host)))