From 0a5e3522a664450eaf0eaa8f0b8a65c63a6fddc9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:41:16 -0500 Subject: [PATCH 01/70] Fix fast-debug in dataservice image entrypoint. Fixing logic where pip installs the updated packages (to avoid the entire image rebuild), so make sure deps are ignored (as this was the slow part). --- docker/main/dataservice/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/dataservice/entrypoint.sh b/docker/main/dataservice/entrypoint.sh index e42ae462d..433ecc801 100644 --- a/docker/main/dataservice/entrypoint.sh +++ b/docker/main/dataservice/entrypoint.sh @@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}') - pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) + pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) fi done #pip install ${UPDATED_PACKAGES_DIR}/*.whl From 82639df55ad49fca9db7bd48d44fd710aa762bb1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 14:43:19 -0500 Subject: [PATCH 02/70] Fix imports in AllocationParadigm unit tests. --- python/lib/core/dmod/test/test_allocationparadigm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/core/dmod/test/test_allocationparadigm.py b/python/lib/core/dmod/test/test_allocationparadigm.py index 060e45f09..d37e36445 100644 --- a/python/lib/core/dmod/test/test_allocationparadigm.py +++ b/python/lib/core/dmod/test/test_allocationparadigm.py @@ -1,6 +1,6 @@ import unittest -from dmod.scheduler.job.job import AllocationParadigm from dmod.communication import SchedulerRequestMessage +from ..core.execution import AllocationParadigm class TestJobAllocationParadigm(unittest.TestCase): From 187dad2a50b78ff930855a313776220de7f1c8e9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:49:07 -0500 Subject: [PATCH 03/70] Add package client name vars to example.env. --- example.env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/example.env b/example.env index dd98d004c..ed8baf63f 100644 --- a/example.env +++ b/example.env @@ -101,6 +101,11 @@ NGEN_BRANCH=master ## Python Packages Settings ## ######################################################################## +## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip) +PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client +## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) +PYTHON_PACKAGE_NAME_CLIENT=dmod.client + ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip) PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) From c7754f73a59fd0dc94813c846f9721a4b2305353 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:52:58 -0500 Subject: [PATCH 04/70] Add new QueryType for dataset management messages. Adding type GET_SERIALIZED_FORM to get the entire serialized state of a dataset. --- .../dmod/communication/dataset_management_message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index d148b6d0e..55f51e7b5 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -16,6 +16,7 @@ class QueryType(Enum): GET_VALUES = 6 GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 + GET_SERIALIZED_FORM = 9 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': From fd5c834fb42f7939a82388eaf8ed7ac7079b12cd Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:37:47 -0500 Subject: [PATCH 05/70] Add GET_SERIALIZED_FORM query dataservice support. --- python/services/dataservice/dmod/dataservice/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 04e1fa0af..60a27cec1 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -686,7 +686,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained {} Items List', data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) - # TODO: (later) add support for messages with other query types also + elif query_type == QueryType.GET_SERIALIZED_FORM: + dataset_name = message.dataset_name + serialized_form = self.get_known_datasets()[dataset_name].to_dict() + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, + reason='Obtained serialized {} dataset'.format(dataset_name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) return DatasetManagementResponse(action=message.management_action, success=False, reason=reason) From 34b1528226a7c425cc22e11c149447ed2f4f1aee Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 10:58:13 -0500 Subject: [PATCH 06/70] Refactor ExternalRequestClient async_make_request. Refactor to ensure it takes advantage of connection handling via its async context manager logic. --- python/lib/communication/dmod/communication/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/client.py b/python/lib/communication/dmod/communication/client.py index aa81b56f3..44fb880fd 100644 --- a/python/lib/communication/dmod/communication/client.py +++ b/python/lib/communication/dmod/communication/client.py @@ -787,10 +787,8 @@ def _update_after_valid_response(self, response: EXTERN_REQ_R): # TODO: this can probably be taken out, as the superclass implementation should suffice async def async_make_request(self, request: EXTERN_REQ_M) -> EXTERN_REQ_R: - async with websockets.connect(self.endpoint_uri, ssl=self.client_ssl_context) as websocket: - await websocket.send(request.to_json()) - response = await websocket.recv() - return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) + response = await self.async_send(request.to_json(), await_response=True) + return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) @property def errors(self): From a8df3632c2dce90a0de2a83d3f828ac4cfb7a3ce Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:43 -0500 Subject: [PATCH 07/70] Adding client methods for data/metadata retrieval. --- .../lib/client/dmod/client/request_clients.py | 126 +++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 09251c01b..9a020119c 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -9,13 +9,13 @@ from dmod.communication.data_transmit_message import DataTransmitMessage, DataTransmitResponse from dmod.core.meta_data import DataCategory, DataDomain, TimeRange from pathlib import Path -from typing import List, Optional, Tuple, Type, Union +from typing import AnyStr, Dict, List, Optional, Tuple, Type, Union import json import websockets -#import logging -#logger = logging.getLogger("gui_log") +import logging +logger = logging.getLogger("client_log") class NgenRequestClient(ModelExecRequestClient[NGENRequest, NGENRequestResponse]): @@ -89,6 +89,25 @@ async def create_dataset(self, name: str, category: DataCategory, domain: DataDo async def delete_dataset(self, name: str, **kwargs) -> bool: pass + @abstractmethod + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + pass + @abstractmethod async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ @@ -130,6 +149,14 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ pass + @abstractmethod + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + pass + + @abstractmethod + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + pass + @abstractmethod async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: pass @@ -175,6 +202,50 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def get_dataset_content_details(self, name: str, **kwargs) -> dict: + # TODO: later add things like created and last updated perhaps + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return {} + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + request = DatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + data_location=item_name, blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ Download an entire dataset to a local directory. @@ -460,6 +531,55 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + await self._async_acquire_session_info() + request = MaaSDatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + session_secret=self.session_secret, data_location=item_name, + blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = MaaSDatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + session_secret=self._session_secret, data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def get_dataset_content_details(self, name: str, **kwargs) -> List: + # TODO: later add things like created and last updated perhaps + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = MaaSDatasetManagementMessage(session_secret=self.session_secret, action=ManagementAction.QUERY, + query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + else: + return [] + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: await self._async_acquire_session_info() try: From 98ab36ad5a5898fdfb0b975da26690085d0e108a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:50:05 -0500 Subject: [PATCH 08/70] Refactor dataset request handler to use try block. --- .../externalrequests/maas_request_handlers.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py index b56f1d10e..fc2b48e09 100644 --- a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py +++ b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py @@ -288,25 +288,28 @@ async def handle_request(self, request: MaaSDatasetManagementMessage, **kwargs) session, is_authorized, reason, msg = await self.get_authorized_session(request) if not is_authorized: return MaaSDatasetManagementResponse(success=False, reason=reason.name, message=msg) - # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) - async with self.service_client as client: - # Have to handle these two slightly differently, since multiple message will be going over the websocket - if request.management_action == ManagementAction.REQUEST_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - elif request.management_action == ManagementAction.ADD_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - else: - mgmt_response = await client.async_make_request(request) - logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) - # Likewise, can just send back the response from the internal service client - return MaaSDatasetManagementResponse.factory_create(mgmt_response) + try: + # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) + async with self.service_client as client: + # Have to handle these two slightly differently, since multiple message will be going over the websocket + if request.management_action == ManagementAction.REQUEST_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + elif request.management_action == ManagementAction.ADD_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + else: + mgmt_response = await client.async_make_request(request) + logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) + # Likewise, can just send back the response from the internal service client + return MaaSDatasetManagementResponse.factory_create(mgmt_response) + except Exception as e: + raise e @property def service_client(self) -> DataServiceClient: if self._service_client is None: - self._service_client = DataServiceClient(self.service_url, self.service_ssl_dir) + self._service_client = DataServiceClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir) return self._service_client From 6e2cf9a652535d9269822a7cc7b9a8eb905e441c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:56:02 -0500 Subject: [PATCH 09/70] Add get_serialized_datasets to external client. Add new function to get serialized dataset details to DatasetExternalClient. --- .../lib/client/dmod/client/request_clients.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 9a020119c..2ea95d98b 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -625,6 +625,37 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P if not has_data: return message_object + async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> Dict[str, dict]: + """ + Get dataset objects in serialized form, either for all datasets or for the one with the provided name. + + Parameters + ---------- + dataset_name : Optional[str] + The name of a specific dataset to get serialized details of, if only one should be obtained. + + Returns + ------- + Dict[str, dict] + A dictionary, keyed by dataset name, of serialized dataset objects. + """ + # TODO: may need to generalize this and add to super class + if dataset_name is None: + datasets = await self.list_datasets() + else: + datasets = [dataset_name] + serialized = dict() + action = ManagementAction.QUERY + query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data['dataset'] + # TODO: what to do if any are not successful + return serialized + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() action = ManagementAction.LIST_ALL if category is None else ManagementAction.SEARCH From 6c7535c38413a86a991255391fad74685878759f Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:01:25 -0500 Subject: [PATCH 10/70] Improve DatasetExternalClient. Fix issue with not acquiring a session, and wrapping some things in try block to catch and log exceptions. --- .../lib/client/dmod/client/request_clients.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 2ea95d98b..bd89d40e7 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -643,18 +643,24 @@ async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> D if dataset_name is None: datasets = await self.list_datasets() else: + # TODO: improve how this is use so that it can be safely, efficiently put everywhere it **may** be needed + await self._async_acquire_session_info() datasets = [dataset_name] serialized = dict() action = ManagementAction.QUERY query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) - for d in datasets: - request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, - session_secret=self.session_secret) - self.last_response: DatasetManagementResponse = await self.async_make_request(request) - if self.last_response.success: - serialized[d] = self.last_response.data['dataset'] - # TODO: what to do if any are not successful - return serialized + try: + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + # TODO: what to do if any are not successful + return serialized + except Exception as e: + logger.error(e) + raise e async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() From c8a0569e69451ba19045854e798e95189b5d39a5 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:55 -0500 Subject: [PATCH 11/70] Bump dataservice dependencies. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index 32dc10aab..b78ca1d01 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.1.1', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.8.0', + install_requires=['dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.8.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) From 839f70e2c4938af77f4d1ac64e00e307515c9cb9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:36:20 -0500 Subject: [PATCH 12/70] Fix LIST_FILES query response bug in dataservice. Fixing bug in response to LIST_FILES query, where reason text was not being assembled entirely correctly. --- python/services/dataservice/dmod/dataservice/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 60a27cec1..7926525d8 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -684,7 +684,7 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement dataset_name = message.dataset_name list_of_files = self.get_known_datasets()[dataset_name].manager.list_files(dataset_name) return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, - reason='Obtained {} Items List', + reason='Obtained {} Items List'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) elif query_type == QueryType.GET_SERIALIZED_FORM: dataset_name = message.dataset_name From 83387f93e88cff53a8d23192ea5575f9ed9e9eb1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:30:53 -0500 Subject: [PATCH 13/70] Update dataset manager abstract interface. Adding optional offset and length params to abstract interface definition of get_data, and adding get_file_stat abstract method. --- .../modeldata/dmod/modeldata/data/dataset.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/dataset.py b/python/lib/modeldata/dmod/modeldata/data/dataset.py index aac52024b..fe9044d52 100644 --- a/python/lib/modeldata/dmod/modeldata/data/dataset.py +++ b/python/lib/modeldata/dmod/modeldata/data/dataset.py @@ -799,7 +799,8 @@ def filter(self, base_dataset: Dataset, restrictions: List[Union[ContinuousRestr pass @abstractmethod - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, Any]: + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> Union[bytes, Any]: """ Get data from this dataset. @@ -812,6 +813,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, The dataset from which to get data. item_name : str The name of the object from which to get data. + offset : Optional[int] + Optional start byte position of object data. + length : Optional[int] + Optional number of bytes of object data from offset. kwargs Implementation-specific params for representing what data to get and how to get and deliver it. @@ -861,6 +866,26 @@ def link_user(self, user: DatasetUser, dataset: Dataset) -> bool: self._dataset_users[dataset.name].add(user.uuid) return True + @abstractmethod + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + pass + @abstractmethod def list_files(self, dataset_name: str, **kwargs) -> List[str]: """ From d06837c0efd9907698665d7fa5e14289cdcbfaa1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:33:28 -0500 Subject: [PATCH 14/70] More QueryType elements and update DatasetQuery. Adding support for querying about dataset items. --- .../dataset_management_message.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 55f51e7b5..73a80b164 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -17,6 +17,10 @@ class QueryType(Enum): GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 GET_SERIALIZED_FORM = 9 + GET_LAST_UPDATED = 10 + GET_SIZE = 11 + GET_ITEM_SIZE = 12 + GET_DATASET_ITEMS = 13 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': @@ -43,26 +47,32 @@ def get_for_name(cls, name_str: str) -> 'QueryType': class DatasetQuery(Serializable): _KEY_QUERY_TYPE = 'query_type' + _KEY_ITEM_NAME = 'item_name' @classmethod def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['DatasetQuery']: try: - return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE])) + return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE]), + item_name=json_obj.get(cls._KEY_ITEM_NAME)) except Exception as e: return None def __hash__(self): - return hash(self.query_type) + return hash('{}{}'.format(self.query_type.name, self.item_name if self.item_name is not None else '')) def __eq__(self, other): - return isinstance(other, DatasetQuery) and self.query_type == other.query_type + return isinstance(other, DatasetQuery) and self.query_type == other.query_type \ + and self.item_name == other.item_name - def __init__(self, query_type: QueryType): + def __init__(self, query_type: QueryType, item_name: Optional[str] = None): self.query_type = query_type + self.item_name = item_name def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial = dict() serial[self._KEY_QUERY_TYPE] = self.query_type.name + if self.item_name is not None: + serial[self._KEY_ITEM_NAME] = self.item_name return serial From 736d3c1c106c1a6c568d58d4fa43a5ddfafb25ac Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:34:17 -0500 Subject: [PATCH 15/70] Update dataset message to support start and size. Updating to support indicating start and size of data for partial transfers. --- .../dataset_management_message.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 73a80b164..d21192906 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -192,6 +192,8 @@ class DatasetManagementMessage(AbstractInitRequest): _SERIAL_KEY_CATEGORY = 'category' _SERIAL_KEY_DATA_DOMAIN = 'data_domain' _SERIAL_KEY_DATA_LOCATION = 'data_location' + _SERIAL_KEY_DATA_BLK_START = 'data_blk_start' + _SERIAL_KEY_DATA_BLK_SIZE = 'data_blk_size' _SERIAL_KEY_DATASET_NAME = 'dataset_name' _SERIAL_KEY_IS_PENDING_DATA = 'pending_data' _SERIAL_KEY_QUERY = 'query' @@ -228,6 +230,8 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase category_str = json_obj.get(cls._SERIAL_KEY_CATEGORY) category = None if category_str is None else DataCategory.get_for_name(category_str) data_loc = json_obj.get(cls._SERIAL_KEY_DATA_LOCATION) + data_blk_start = json_obj.get(cls._SERIAL_KEY_DATA_BLK_START) + data_blk_size = json_obj.get(cls._SERIAL_KEY_DATA_BLK_SIZE) #page = json_obj[cls._SERIAL_KEY_PAGE] if cls._SERIAL_KEY_PAGE in json_obj else None if cls._SERIAL_KEY_QUERY in json_obj: query = DatasetQuery.factory_init_from_deserialized_json(json_obj[cls._SERIAL_KEY_QUERY]) @@ -240,7 +244,7 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase return deserialized_class(action=action, dataset_name=dataset_name, category=category, is_read_only_dataset=json_obj[cls._SERIAL_KEY_IS_READ_ONLY], domain=domain, - data_location=data_loc, + data_location=data_loc, blk_start=data_blk_start, blk_size=data_blk_size, is_pending_data=json_obj.get(cls._SERIAL_KEY_IS_PENDING_DATA), #page=page, query=query, **deserialized_class_kwargs) except Exception as e: @@ -272,8 +276,8 @@ def __hash__(self): def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, is_read_only_dataset: bool = False, category: Optional[DataCategory] = None, domain: Optional[DataDomain] = None, - data_location: Optional[str] = None, is_pending_data: bool = False, - query: Optional[DatasetQuery] = None, *args, **kwargs): + data_location: Optional[str] = None, blk_start: Optional[int] = None, blk_size: Optional[int] = None, + is_pending_data: bool = False, query: Optional[DatasetQuery] = None, *args, **kwargs): """ Initialize this instance. @@ -289,6 +293,10 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, The optional category of the involved dataset or datasets, when applicable; defaults to ``None``. data_location : Optional[str] Optional location/file/object/etc. for acted-upon data. + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool Whether the sender has data pending transmission after this message (default: ``False``). query : Optional[DatasetQuery] @@ -313,9 +321,19 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, self._category = category self._domain = domain self._data_location = data_location + self._blk_start = blk_start + self._blk_size = blk_size self._query = query self._is_pending_data = is_pending_data + @property + def blk_size(self) -> Optional[int]: + return self._blk_size + + @property + def blk_start(self) -> Optional[int]: + return self._blk_start + @property def data_location(self) -> Optional[str]: """ @@ -417,6 +435,10 @@ def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial[self._SERIAL_KEY_CATEGORY] = self.data_category.name if self.data_location is not None: serial[self._SERIAL_KEY_DATA_LOCATION] = self.data_location + if self._blk_start is not None: + serial[self._SERIAL_KEY_DATA_BLK_START] = self._blk_start + if self._blk_size is not None: + serial[self._SERIAL_KEY_DATA_BLK_SIZE] = self._blk_size if self.data_domain is not None: serial[self._SERIAL_KEY_DATA_DOMAIN] = self.data_domain.to_dict() if self.query is not None: @@ -613,6 +635,10 @@ def __init__(self, session_secret: str, *args, **kwargs): is_read_only_dataset : bool category : Optional[DataCategory] data_location : Optional[str] + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool query : Optional[DataQuery] """ From 4c5e1e4896bf81acd752ddccf933f604a7ddcb22 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:58:11 -0500 Subject: [PATCH 16/70] Bump requestservice dependencies. --- python/services/requestservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/setup.py b/python/services/requestservice/setup.py index 06618889d..af7ed1e0a 100644 --- a/python/services/requestservice/setup.py +++ b/python/services/requestservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=0.7.0', 'dmod-access>=0.2.0', + install_requires=['websockets', 'dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-access>=0.2.0', 'dmod-externalrequests>=0.3.0'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) From 62a29986c7bd79e417a4038b2939178e74330e76 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:33 -0500 Subject: [PATCH 17/70] Bump requestservice to 0.6.0. --- python/services/requestservice/dmod/requestservice/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/dmod/requestservice/_version.py b/python/services/requestservice/dmod/requestservice/_version.py index 9bdd4d277..83e147c62 100644 --- a/python/services/requestservice/dmod/requestservice/_version.py +++ b/python/services/requestservice/dmod/requestservice/_version.py @@ -1 +1 @@ -__version__ = '0.5.0' \ No newline at end of file +__version__ = '0.6.0' \ No newline at end of file From a6bce6a43be0d24d377aeb4f370e6cca837655ff Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:35:21 -0500 Subject: [PATCH 18/70] Bump communication package version to 0.9.1. --- python/lib/communication/dmod/communication/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/communication/dmod/communication/_version.py b/python/lib/communication/dmod/communication/_version.py index e4e49b3bb..8969d4966 100644 --- a/python/lib/communication/dmod/communication/_version.py +++ b/python/lib/communication/dmod/communication/_version.py @@ -1 +1 @@ -__version__ = '0.9.0' +__version__ = '0.9.1' From ea99c831d08d00a97ff70752d14af0cf5f53f49d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:41:13 -0500 Subject: [PATCH 19/70] Optimize object store dataset reloading. Optimizing the reloading of datasets on object store manager startup. --- .../dmod/modeldata/data/object_store_manager.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 084492805..2ee09dcd3 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -61,8 +61,13 @@ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, se # For any buckets that have the standard serialized object (i.e., were for datasets previously), reload them for bucket_name in self.list_buckets(): serialized_item = self._gen_dataset_serial_obj_name(bucket_name) - if serialized_item in [o.object_name for o in self._client.list_objects(bucket_name)]: + try: self.reload(reload_from=bucket_name, serialized_item=serialized_item) + except minio.error.S3Error as e: + # Continue with looping through buckets and initializing if we get this particular exception and + # error code, but otherwise pass through the exception + if e.code != "NoSuchKey": + raise e except Exception as e: self._errors.append(e) # TODO: consider if we should not re-throw this (which would likely force us to ensure users checked this) @@ -554,12 +559,14 @@ def reload(self, reload_from: str, serialized_item: Optional[str] = None) -> Dat if serialized_item is None: serialized_item = self._gen_dataset_serial_obj_name(reload_from) + response_obj = None try: response_obj = self._client.get_object(bucket_name=reload_from, object_name=serialized_item) response_data = json.loads(response_obj.data.decode()) finally: - response_obj.close() - response_obj.release_conn() + if response_obj is not None: + response_obj.close() + response_obj.release_conn() # If we can safely infer it, make sure the "type" key is set in cases when it is missing if len(self.supported_dataset_types) == 1 and Dataset._KEY_TYPE not in response_data: From bb629a4459884ea421e4bd0eae4a047856e8e90c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:42:39 -0500 Subject: [PATCH 20/70] Update object store manager for interface changes. Accounting for changes to get_data parameters and implementing get_file_stat. --- .../modeldata/data/object_store_manager.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 2ee09dcd3..a33e5fc29 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -10,7 +10,7 @@ from minio.api import ObjectWriteResult from minio.deleteobjects import DeleteObject from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from uuid import UUID @@ -425,7 +425,32 @@ def delete_data(self, dataset_name: str, **kwargs) -> bool: self._errors.extend(error_list) return False - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + obj_stat = self._client.stat_object(dataset_name, file_name) + as_dict = dict() + as_dict["name"] = obj_stat.object_name + as_dict["size"] = obj_stat.size + # TODO: get more of this if worth it + return as_dict + + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> bytes: """ Get data from this dataset. @@ -439,15 +464,12 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: The name of the dataset (i.e., bucket) from which to get data. item_name : str The name of the object from which to get data. - kwargs - Implementation-specific params for representing what data to get and how to get and deliver it. - - Keyword Args - ------- - offset : int + offset : Optional[int] Optional start byte position of object data. - length : int + length : Optional[int] Optional number of bytes of object data from offset. + kwargs + Implementation-specific params for representing what data to get and how to get and deliver it. Returns ------- @@ -457,8 +479,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: if item_name not in self.list_files(dataset_name): raise RuntimeError('Cannot get data for non-existing {} file in {} dataset'.format(item_name, dataset_name)) optional_params = dict() - for key in [k for k in self.data_chunking_params if k in kwargs]: - optional_params[key] = kwargs[key] + if offset is not None: + optional_params['offset'] = offset + if length is not None: + optional_params['length'] = length response_object = self._client.get_object(bucket_name=dataset_name, object_name=item_name, **optional_params) return response_object.data From 39c36250fe2f8825cd0a141c49f8189c2c490497 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:35:57 -0500 Subject: [PATCH 21/70] Update modeldata dependency versions. --- python/lib/modeldata/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index c536a6b03..69bf05c84 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.1.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.9.1', 'dmod-core>=0.2.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) From 4ea065b74cdc164a6b845c1e1c676931c0b50227 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:36:06 -0500 Subject: [PATCH 22/70] Bump modeldata package version to 0.9.0. --- python/lib/modeldata/dmod/modeldata/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/dmod/modeldata/_version.py b/python/lib/modeldata/dmod/modeldata/_version.py index ccf9e6286..1658609d0 100644 --- a/python/lib/modeldata/dmod/modeldata/_version.py +++ b/python/lib/modeldata/dmod/modeldata/_version.py @@ -1 +1 @@ -__version__ = '0.8.0' \ No newline at end of file +__version__ = '0.9.0' \ No newline at end of file From 5f6e3899fd5c8c6bb1f86c580e3824c50e5608ac Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 14:32:19 -0500 Subject: [PATCH 23/70] Update client unit tests for interface changes. --- .../client/dmod/test/test_dataset_client.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/lib/client/dmod/test/test_dataset_client.py b/python/lib/client/dmod/test/test_dataset_client.py index b266658c7..37ff6e386 100644 --- a/python/lib/client/dmod/test/test_dataset_client.py +++ b/python/lib/client/dmod/test/test_dataset_client.py @@ -1,7 +1,7 @@ import unittest from ..client.request_clients import DataCategory, DatasetClient, DatasetManagementResponse, MaaSDatasetManagementResponse from pathlib import Path -from typing import List, Optional +from typing import List, Optional, AnyStr class SimpleMockDatasetClient(DatasetClient): @@ -28,6 +28,24 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ Mock implementation, always returning ``False``. """ return False + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Mock implementation, always returning empty string. + """ + return '' + + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + """ + Mock implementation, always returning ``False``. + """ + return False + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + """ + Mock implementation always returning ``1``. + """ + return 1 + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: """ Mock implementation, always returning an empty list. """ return [] From c25d8281f0cc48f1f36f186c065bad916a17ce3c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:54 -0500 Subject: [PATCH 24/70] Bump client package version to 0.2.0. --- python/lib/client/dmod/client/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/client/dmod/client/_version.py b/python/lib/client/dmod/client/_version.py index b794fd409..7fd229a32 100644 --- a/python/lib/client/dmod/client/_version.py +++ b/python/lib/client/dmod/client/_version.py @@ -1 +1 @@ -__version__ = '0.1.0' +__version__ = '0.2.0' From 128b6cd26823bfbc9aad6ba5f1f43be929b1f03f Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:21 -0500 Subject: [PATCH 25/70] Update dataservice response handling. Updating service to respond to GET_DATASET_ITEMS queries and requests for data with an offset start (i.e. partials). --- .../dataservice/dmod/dataservice/service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 7926525d8..1d1c7c37d 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -692,6 +692,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained serialized {} dataset'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + if query_type == QueryType.GET_DATASET_ITEMS: + dataset = self.get_known_datasets()[message.dataset_name] + mgr = dataset.manager + item_details: List[dict] = [mgr.get_file_stat(dataset.name, f) for f in mgr.list_files(dataset.name)] + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset.name, + reason='Obtained file details for {} dataset'.format(dataset.name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: item_details}) # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) @@ -896,6 +903,14 @@ async def listener(self, websocket: WebSocketServerProtocol, path): partial_indx = 0 elif inbound_message.management_action == ManagementAction.CREATE: response = await self._async_process_dataset_create(message=inbound_message) + elif inbound_message.management_action == ManagementAction.REQUEST_DATA and inbound_message.blk_start is not None: + manager = self.get_known_datasets()[inbound_message.dataset_name].manager + raw_data = manager.get_data(dataset_name=inbound_message.dataset_name, + item_name=inbound_message.data_location, + offset=inbound_message.blk_start, length=inbound_message.blk_size) + response = DatasetManagementResponse(success=raw_data is not None, + action=inbound_message.management_action, + data=raw_data, reason="Data Block Retrieve Complete") elif inbound_message.management_action == ManagementAction.REQUEST_DATA: response = await self._async_process_data_request(message=inbound_message, websocket=websocket) elif inbound_message.management_action == ManagementAction.ADD_DATA: From 791b1c23ec8116dccd46a25cafd6a1a5f8a40338 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:46 -0500 Subject: [PATCH 26/70] Update dataservice package dependency versions. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index b78ca1d01..1d87d90f7 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.8.0', + install_requires=['dmod-core>=0.2.0', 'dmod-communication>=0.9.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) From 39f369acfc57bc9b23ccacb2785f71dafb62299a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:39:04 -0500 Subject: [PATCH 27/70] Bump dataservice version to 0.4.0. --- python/services/dataservice/dmod/dataservice/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/_version.py b/python/services/dataservice/dmod/dataservice/_version.py index 290d7c60d..222c11cfd 100644 --- a/python/services/dataservice/dmod/dataservice/_version.py +++ b/python/services/dataservice/dmod/dataservice/_version.py @@ -1 +1 @@ -__version__ = '0.3.0' \ No newline at end of file +__version__ = '0.4.0' \ No newline at end of file From cb5487912693f179709a32b595586945b7be580d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:46:52 -0500 Subject: [PATCH 28/70] Add dataset view class and static page for GUI. --- python/gui/MaaS/cbv/DatasetManagementView.py | 132 +++++++ .../templates/maas/dataset_management.html | 359 ++++++++++++++++++ 2 files changed, 491 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetManagementView.py create mode 100644 python/gui/MaaS/templates/maas/dataset_management.html diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py new file mode 100644 index 000000000..bfaf7fd10 --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -0,0 +1,132 @@ +""" +Defines a view that may be used to configure a MaaS request +""" +import asyncio +import os +from django.http import HttpRequest, HttpResponse +from django.views.generic.base import View +from django.shortcuts import render + +import dmod.communication as communication +from dmod.client.request_clients import DatasetExternalClient +from dmod.core.meta_data import DataCategory, DataFormat + +import logging +logger = logging.getLogger("gui_log") + +from pathlib import Path + +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from .utils import extract_log_data +from typing import Dict + + +class DatasetManagementView(View, DMODMixin): + + """ + A view used to configure a dataset management request or requests for transmitting dataset data. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._dataset_client = None + + def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: + """ + Determine and return whether this request is for a ``DATASET_MANAGEMENT`` or ``DATA_TRANSMISSION`` event. + + Parameters + ---------- + http_request : HttpRequest + The raw HTTP request in question. + + Returns + ------- + communication.MessageEventType + Either ``communication.MessageEventType.DATASET_MANAGEMENT`` or + ``communication.MessageEventType.DATA_TRANSMISSION``. + """ + # TODO: + raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client + + async def _get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'get' requests. + + This will render the 'maas/dataset_management.html' template after retrieving necessary information to initially + populate the forms it displays. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + errors, warnings, info = extract_log_data(kwargs) + + # Gather map of serialized datasets, keyed by dataset name + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + + dataset_categories = [c.name.title() for c in DataCategory] + dataset_formats = [f.name for f in DataFormat] + + payload = { + 'datasets': serial_dataset_map, + 'dataset_categories': dataset_categories, + 'dataset_formats': dataset_formats, + 'errors': errors, + 'info': info, + 'warnings': warnings + } + + # TODO: create this file + return render(http_request, 'maas/dataset_management.html', payload) + + def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'post' requests. + + This will attempt to submit the request and rerender the page like a 'get' request. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + # TODO: implement this to figure out whether DATASET_MANAGEMENT or DATA_TRANSMISSION + event_type = self._process_event_type(http_request) + client, session_data, dmod_response = self.forward_request(http_request, event_type) + + # TODO: this probably isn't exactly correct, so review once closer to completion + if dmod_response is not None and 'dataset_id' in dmod_response.data: + session_data['new_dataset_id'] = dmod_response.data['dataset_id'] + + http_response = self.get(http_request=http_request, errors=client.errors, warnings=client.warnings, + info=client.info, *args, **kwargs) + + for k, v in session_data.items(): + http_response.set_cookie(k, v) + + return http_response diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html new file mode 100644 index 000000000..9ac763131 --- /dev/null +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -0,0 +1,359 @@ + + + + + OWP MaaS + {% load static %} + + + + + + + + +
+ + + {% if errors %} +
+
    + {% for error in errors %} +
  • {{ error }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if warnings %} +
+
    + {% for warning in warnings %} +
  • {{ warning }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if info %} +
+
    + {% for message in info %} +
  • {{ message }}
    • + {% endfor %} +
+
+ {% endif %} + + {# Cache jQuery scripts for UI scripting and styling #} +
+

Dataset Management

+
+ Manage + Create +
+
+

Create New Dataset:

+
+ {# Add the token to provide cross site request forgery protection #} + {% csrf_token %} +
+ + +
+ + + + +
+ + + + +
+ + +
+
+ + +
+
+
+
+ + From a6238f260e6f2da5ab7f1563eb2c1d855bdc1353 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:47:49 -0500 Subject: [PATCH 29/70] Refactor static dir usage in DMODProxy.py. --- python/gui/MaaS/cbv/DMODProxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DMODProxy.py b/python/gui/MaaS/cbv/DMODProxy.py index fc0fcb7a1..e9e308664 100644 --- a/python/gui/MaaS/cbv/DMODProxy.py +++ b/python/gui/MaaS/cbv/DMODProxy.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import List, Optional, Tuple, Type +GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl') + class RequestFormProcessor(ABC): @@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient): def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None): if ssl_dir is None: ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl') - ssl_dir = Path('/usr/maas_portal/ssl') #Fixme + ssl_dir = GUI_STATIC_SSL_DIR #Fixme return ssl_dir def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None): @@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) -> client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request) if event_type == MessageEventType.MODEL_EXEC_REQUEST: form_processor_type = ModelExecRequestFormProcessor + # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading) else: raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type))) From 0758a385eec6509f706524821cea0eb297a24682 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:59:42 -0500 Subject: [PATCH 30/70] Add Django GUI url def for dataset view. --- python/gui/MaaS/urls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index 704af44b3..c22dcac29 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,5 +1,6 @@ from django.conf.urls import url from .cbv.EditView import EditView +from .cbv.DatasetManagementView import DatasetManagementView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -10,6 +11,9 @@ urlpatterns = [ url(r'^$', EditView.as_view()), + # TODO: add this later + #url(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), + url(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), url(r'map$', MapView.as_view(), name="map"), url(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), url(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From 28215fc04877a88ec9df786eb7ba1607c0a2920f Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:48:19 -0500 Subject: [PATCH 31/70] Add client package build arg for GUI Docker build. --- docker/nwm_gui/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 509441649..97c7a7b68 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -34,6 +34,7 @@ services: args: docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)} comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?} + client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?} networks: - request-listener-net # Call this when starting the container From 39d6c518788fab79e36ace13ee9c213f9af020d4 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 29 Sep 2022 10:38:26 -0500 Subject: [PATCH 32/70] Add second view and navigation. Adding initial second view display and manage existing datasets, along with navigation functionality to toggle between "manage" and "create" views. --- .../templates/maas/dataset_management.html | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 9ac763131..c2fcd1827 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,20 @@ $("#" + model + "_parameters").show(); } + /** + * Toggle what is displayed based on selection click in main nav bar for page. + * + * @param {string} selected_div_id - The id of the main-level div selected to be displayed. + */ + function toggleNav(selected_div_id) { + const all_nav_divs = ["dataset-manage", "dataset-create"]; + for (let i = 0; i <= all_nav_divs.length; i++) { + // TODO: consider a warning message for receiving an unexpected selection + let nav_div = document.getElementById(all_nav_divs[i]); + nav_div.style.display = (selected_div_id === all_nav_divs[i]) ? 'block' : 'none'; + } + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -152,6 +166,14 @@ font-size: 17px; } + #dataset-manage { + display: block; + } + + #dataset-create { + display: none; + } + @@ -196,8 +218,79 @@

Office of Water Prediction - Model as a Service

Dataset Management

- Manage - Create + Manage + Create +
+
+ + + + + + + + + + + {% for dataset in datasets %} + + + + + + + + + + {% endfor %} +
DatasetCategory
{{ dataset.name }}{{ dataset.data_category }}
+ + {% for dataset in datasets %} +
+ + +
+ × + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
+
+
+ {% endfor %}

Create New Dataset:

From 45f0b4f760b6c7b9c97915b8425973ad39cdf6b7 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:11:10 -0500 Subject: [PATCH 33/70] Have DatasetManagementView.py send DS as list. Sending serialized datasets to HTML template as list/array rather than dict/map. --- python/gui/MaaS/cbv/DatasetManagementView.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index bfaf7fd10..47f411888 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -82,12 +82,13 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: # Gather map of serialized datasets, keyed by dataset name serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] dataset_formats = [f.name for f in DataFormat] payload = { - 'datasets': serial_dataset_map, + 'datasets': serial_dataset_list, 'dataset_categories': dataset_categories, 'dataset_formats': dataset_formats, 'errors': errors, From 59743c57e90feccd72f6339d77bc92050ef2886e Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:15:18 -0500 Subject: [PATCH 34/70] Update dataset management GUI with details view. Implementing layout and initial details-viewing behavior for dataset management GUI view. --- .../templates/maas/dataset_management.html | 154 +++++++++++++++++- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index c2fcd1827..f54c714f4 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -30,6 +30,125 @@ } } + function buildDatasetDetailsEmptyDiv() { + let details_div = document.createElement('div'); + details_div.id = 'details-div'; + details_div.class = 'details-modal'; + details_div.style.position = 'fixed'; + details_div.style.zIndex = '1'; + details_div.style.left = '25%'; + details_div.style.top = '5%'; + details_div.style.width = '50%'; + details_div.style.height = '50%'; + details_div.style.overflow = 'clip'; + details_div.style.backgroundColor = '#B7B5B5FF'; + details_div.style.border = '1px solid #888'; + details_div.style.padding = '5px'; + details_div.style.paddingTop = '0px'; + details_div.style.margin = '15% auto'; + + let close_x = document.createElement('span'); + close_x.style.padding = "0px"; + close_x.style.cursor = "pointer"; + close_x.style.fontWeight = "bold"; + close_x.style.fontSize = "20px"; + close_x.appendChild(document.createTextNode("\u00d7")); + close_x.onclick = function() { details_div.remove() }; + details_div.appendChild(close_x); + + let details_content_div = document.createElement('div'); + details_content_div.style.height = '90%'; + details_content_div.style.overflow = 'auto'; + + details_div.appendChild(details_content_div); + + let parent_div = document.getElementById('dataset-manage'); + parent_div.appendChild(details_div); + return details_content_div; + } + + function buildDetailsTableRow() { + let row = document.createElement('tr'); + // Skip the first arg, as this is the table itself + for (let i = 1; i < arguments.length; i++) { + let colCell = document.createElement('th'); + if (i == 1) { + colCell.style.textAlign = 'right'; + colCell.style.paddingRight = '8px'; + arguments[i] += ":"; + } + else { + colCell.style.textAlign = 'left'; + colCell.style.fontWeight = 'normal'; + } + let cellText = document.createTextNode(arguments[i]); + colCell.appendChild(cellText); + row.appendChild(colCell); + } + arguments[0].appendChild(row); + } + + function buildDatasetDetailsTable(dataset_name) { + {% for dataset in datasets %} + if (dataset_name == "{{ dataset.name }}") { + //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); + //return cellText; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + buildDetailsTableRow(table, "Name", "{{ dataset.name }}") + buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") + buildDetailsTableRow(table, "Type", "{{ dataset.type }}") + buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") + buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") + buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") + buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + {% for cont_rest in dataset.data_domain.continuous %} + buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); + {% endfor %} + {% for disc_rest in dataset.data_domain.discrete %} + buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); + {% endfor %} + + return table; + } + {% endfor %} + } + + function displayDatasetDetails(name) { + let details_div = buildDatasetDetailsEmptyDiv(); + + let detailsTable = buildDatasetDetailsTable(name); + details_div.appendChild(detailsTable); + details_div.style.display = 'block'; + } + + function clickDownloadDataset(dataset_name) { + return false; + } + + function clickUploadDataset(dataset_name) { + return false; + } + + function clickDeleteDataset(dataset_name) { + return false; + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -170,6 +289,25 @@ display: block; } + .datasets-table thead tr th { + padding-bottom: 5px; + } + + .mgr-tbl-dataset-header { + text-align: left; + } + + .mgr-tbl-category-header { + text-align: left; + } + + .mgr-tbl-content th { + font-weight: normal; + text-align: left; + padding-right: 15px; + padding-bottom: 2px; + } + #dataset-create { display: none; } @@ -225,8 +363,10 @@

Dataset Management

- - + + + + - + + + + + {% for dataset in datasets %} @@ -386,57 +380,9 @@

Dataset Management

- - - - {% endfor %}
DatasetCategoryDataset NameCategoryActions Details
- - {% for dataset in datasets %} -
- - -
- × - - - - - - - - - - - - - - - - - - - - - - - - - - -
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
-
-
- {% endfor %}

Create New Dataset:

From 4aeab61d3a13e008b9585c76de373f9ef575cc0a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:42:30 -0500 Subject: [PATCH 36/70] Create AbstractDatasetView GUI Django view class. --- python/gui/MaaS/cbv/AbstractDatasetView.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/AbstractDatasetView.py diff --git a/python/gui/MaaS/cbv/AbstractDatasetView.py b/python/gui/MaaS/cbv/AbstractDatasetView.py new file mode 100644 index 000000000..e1ae493c3 --- /dev/null +++ b/python/gui/MaaS/cbv/AbstractDatasetView.py @@ -0,0 +1,29 @@ +from abc import ABC +from django.views.generic.base import View +from dmod.client.request_clients import DatasetExternalClient +import logging +logger = logging.getLogger("gui_log") +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from typing import Dict + + +class AbstractDatasetView(View, DMODMixin, ABC): + + def __init__(self, *args, **kwargs): + super(AbstractDatasetView, self).__init__(*args, **kwargs) + self._dataset_client = None + + async def get_dataset(self, dataset_name: str) -> Dict[str, dict]: + serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name) + return serial_dataset + + async def get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client From af45047a79a7408a93ec53a4fd1fb99fadbb45ee Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:15 -0500 Subject: [PATCH 37/70] Add Django view for direct dataservice API calls. --- python/gui/MaaS/cbv/DatasetApiView.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetApiView.py diff --git a/python/gui/MaaS/cbv/DatasetApiView.py b/python/gui/MaaS/cbv/DatasetApiView.py new file mode 100644 index 000000000..8697def2d --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetApiView.py @@ -0,0 +1,29 @@ +import asyncio +from django.http import JsonResponse +from .AbstractDatasetView import AbstractDatasetView +import logging +logger = logging.getLogger("gui_log") + + +class DatasetApiView(AbstractDatasetView): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _get_datasets_json(self) -> JsonResponse: + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) + return JsonResponse({"datasets": serial_dataset_map}, status=200) + + def _get_dataset_json(self, dataset_name: str) -> JsonResponse: + serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name)) + return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200) + + def get(self, request, *args, **kwargs): + request_type = request.GET.get("request_type", None) + if request_type == 'datasets': + return self._get_datasets_json() + elif request_type == 'dataset': + return self._get_dataset_json(dataset_name=request.GET.get("name", None)) + + # TODO: finish + return JsonResponse({}, status=400) From 5a5f3a812549aca4ca3939d8d5392243beee6478 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:31 -0500 Subject: [PATCH 38/70] Add GUI Django url for Dataset API AJAX calls. --- python/gui/MaaS/urls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index c22dcac29..8771c1419 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,6 +1,7 @@ from django.conf.urls import url from .cbv.EditView import EditView from .cbv.DatasetManagementView import DatasetManagementView +from .cbv.DatasetApiView import DatasetApiView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -14,6 +15,7 @@ # TODO: add this later #url(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), url(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), + url(r'dataset-api', DatasetApiView.as_view(), name="dataset-api"), url(r'map$', MapView.as_view(), name="map"), url(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), url(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From cf9de3beec5b5bb5274c913f59c569dae44aea91 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:46:21 -0500 Subject: [PATCH 39/70] Add commented-out helper debug volume for GUI. Adding commented-out line for host bind-mount volume to GUI Docker service config to make debugging Django template and Javascript code faster by mounted code from the host into the running service. --- docker/nwm_gui/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 97c7a7b68..7eb8ef4fb 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -63,6 +63,8 @@ services: volumes: - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl + # Needed only for speeding debugging + #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah} # Expose Django's port to the internal network so that the web server may access it expose: From da982c4343373771f4afc664a1fc05e9ed7569e9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:48:48 -0500 Subject: [PATCH 40/70] Refactor DatasetManagementView inheritance. Make DatasetManagementView inherit from AbstractDatasetView so certain required pieces could be centrally located and reused. --- python/gui/MaaS/cbv/DatasetManagementView.py | 25 +++----------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index 47f411888..b7db81112 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -2,26 +2,20 @@ Defines a view that may be used to configure a MaaS request """ import asyncio -import os from django.http import HttpRequest, HttpResponse -from django.views.generic.base import View from django.shortcuts import render import dmod.communication as communication -from dmod.client.request_clients import DatasetExternalClient from dmod.core.meta_data import DataCategory, DataFormat import logging logger = logging.getLogger("gui_log") -from pathlib import Path - -from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR from .utils import extract_log_data -from typing import Dict +from .AbstractDatasetView import AbstractDatasetView -class DatasetManagementView(View, DMODMixin): +class DatasetManagementView(AbstractDatasetView): """ A view used to configure a dataset management request or requests for transmitting dataset data. @@ -29,7 +23,6 @@ class DatasetManagementView(View, DMODMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._dataset_client = None def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: """ @@ -49,17 +42,6 @@ def _process_event_type(self, http_request: HttpRequest) -> communication.Messag # TODO: raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) - @property - def dataset_client(self) -> DatasetExternalClient: - if self._dataset_client is None: - self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, - ssl_directory=GUI_STATIC_SSL_DIR) - return self._dataset_client - - async def _get_datasets(self) -> Dict[str, dict]: - serial_datasets = await self.dataset_client.get_serialized_datasets() - return serial_datasets - def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: """ The handler for 'get' requests. @@ -81,7 +63,7 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: errors, warnings, info = extract_log_data(kwargs) # Gather map of serialized datasets, keyed by dataset name - serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] @@ -96,7 +78,6 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: 'warnings': warnings } - # TODO: create this file return render(http_request, 'maas/dataset_management.html', payload) def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: From edbfd8d5b21a75de2040414a9de3c826108e6cdf Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:51:29 -0500 Subject: [PATCH 41/70] More progress on dataset management template. --- .../templates/maas/dataset_management.html | 287 +++++++++++++----- 1 file changed, 215 insertions(+), 72 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 1ca18311c..97c694b73 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,143 @@ $("#" + model + "_parameters").show(); } + /** + * Build ``thead`` for the management overview table summarizing datasets. + */ + function buildDatasetsOverviewTableHeader() { + // Build the table header row + let thead = document.createElement('thead'); + let header = document.createElement('tr'); + thead.appendChild(header); + + let colCell = document.createElement('th'); + colCell.className = "mgr-tbl-dataset-header"; + colCell.appendChild(document.createTextNode('Dataset Name')); + header.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.className = "mgr-tbl-category-header"; + colCell.appendChild(document.createTextNode('Category')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode('Actions')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + header.appendChild(document.createElement('th')); + + return thead; + } + + function buildDatasetOverviewTableRow(serial_dataset) { + let row = document.createElement('tr'); + row.className = "mgr-tbl-content"; + + let colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['name'])); + row.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['data_category'])); + row.appendChild(colCell); + + let helper_create_links = function create_links_func (is_a, text, ds_name, on_click) { + let cell = document.createElement('th'); + let content; + if (is_a) { + content = document.createElement('a') + content.href = "javascript:void(0);"; + } + else { + content = document.createElement('button'); + } + content.onclick = function() { on_click(ds_name); }; + content.appendChild(document.createTextNode(text)); + cell.appendChild(content); + row.appendChild(cell); + } + + helper_create_links(true, "Details", serial_dataset['name'], displayDatasetDetails); + helper_create_links(true, "Download", serial_dataset['name'], clickDownloadDataset); + helper_create_links(true, "Upload Files", serial_dataset['name'], clickUploadDataset); + helper_create_links(true, "Delete", serial_dataset['name'], clickDeleteDataset); + + return row; + } + + function buildDatasetsOverviewTable() { + let table_id = 'dataset-manage-overview-table'; + let parent_div = document.getElementById('dataset-manage'); + + // First find and remove any existing table, in case we are refreshing + let table = document.getElementById(table_id); + if (table != null) { + table.remove(); + } + + table = document.createElement('table'); + parent_div.appendChild(table); + table.id = table_id; + table.className = 'datasets-table'; + + table.appendChild(buildDatasetsOverviewTableHeader()); + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "datasets"}, + success: function(response) { + for (const ds_name in response["datasets"]) { + table.appendChild(buildDatasetOverviewTableRow(response["datasets"][ds_name])); + } + }, + error: function(response) { + alert('Received an error'); + console.log(response); + } + }); + } + + function initDatasetCreateForm() { + let category_select = document.getElementById("create-dataset-category"); + category_select.selectedIndex = -1; + let format_select = document.getElementById("create-dataset-format"); + format_select.selectedIndex = -1; + } + + function initGuiViews() { + buildDatasetsOverviewTable(); + initDatasetCreateForm(); + } + + function changeCreateDatasetFormatSelection(selection) { + let dy_div = document.getElementById("dataset-create-form-dynamic-vars-div"); + for (let i = 0; i < dy_div.children.length; i++) { + dy_div.children[i].remove(); + } + let addUploadSelection = false; + if (selection === "NETCDF_FORCING_CANONICAL") { + addUploadSelection = true; + } + + if (addUploadSelection) { + let upload_select_label = document.createElement('label'); + upload_select_label.appendChild(document.createTextNode('Data Files:')); + dy_div.appendChild(upload_select_label); + upload_select_label.htmlFor = 'create-dataset-upload'; + let upload_select = document.createElement('input'); + upload_select.type = 'file'; + upload_select.name = 'create-dataset-upload'; + upload_select.id = 'create-dataset-upload'; + upload_select.style.float = 'right'; + upload_select.style.textAlign = 'right'; + dy_div.appendChild(upload_select); + } + } + /** * Toggle what is displayed based on selection click in main nav bar for page. * @@ -70,9 +207,12 @@ function buildDetailsTableRow() { let row = document.createElement('tr'); // Skip the first arg, as this is the table itself + let colCell; + let cellText; for (let i = 1; i < arguments.length; i++) { - let colCell = document.createElement('th'); + colCell = document.createElement('th'); if (i == 1) { + colCell.style.verticalAlign = 'top'; colCell.style.textAlign = 'right'; colCell.style.paddingRight = '8px'; arguments[i] += ":"; @@ -81,52 +221,69 @@ colCell.style.textAlign = 'left'; colCell.style.fontWeight = 'normal'; } - let cellText = document.createTextNode(arguments[i]); + cellText = document.createTextNode(arguments[i]); colCell.appendChild(cellText); row.appendChild(colCell); } arguments[0].appendChild(row); } + function ajaxBuildDatasetDetailsTable(dataset_json, table) { + buildDetailsTableRow(table, "Name", dataset_json["name"]); + buildDetailsTableRow(table, "Category", dataset_json["data_category"]) + buildDetailsTableRow(table, "Type", dataset_json["type"]); + buildDetailsTableRow(table, "Format", dataset_json["data_domain"]["data_format"]); + buildDetailsTableRow(table, "Read Only", dataset_json["is_read_only"]); + buildDetailsTableRow(table, "Created", dataset_json["create_on"]); + buildDetailsTableRow(table, "Last Updated", dataset_json["last_updated"]); + + // TODO: add something for dataset size + // TODO: add something for number of files. + // TODO: add something for indicating if data aligns properly with configured domain + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + let c_restrict; + for (let c_i = 0; c_i < dataset_json["data_domain"]["continuous"].length; c_i++) { + c_restrict = dataset_json["data_domain"]["continuous"][c_i]; + buildDetailsTableRow(table, c_restrict["variable"], c_restrict["begin"] + " to " + c_restrict["end"]); + } + let d_restrict; + for (let d_i = 0; d_i < dataset_json["data_domain"]["discrete"].length; d_i++) { + d_restrict = dataset_json["data_domain"]["discrete"][d_i]; + buildDetailsTableRow(table, d_restrict["variable"], d_restrict["values"]); + } + } + function buildDatasetDetailsTable(dataset_name) { - {% for dataset in datasets %} - if (dataset_name == "{{ dataset.name }}") { - //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); - //return cellText; - let table = document.createElement('table'); - table.class = 'datasets-details-table'; - buildDetailsTableRow(table, "Name", "{{ dataset.name }}") - buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") - buildDetailsTableRow(table, "Type", "{{ dataset.type }}") - buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") - buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") - buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") - buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") - - // Put spacer row in to indicate the start of domain details - let domain_header_row = document.createElement('tr'); - let colCell = document.createElement('th'); - colCell.colSpan = 2; - colCell.style.textAlign = 'left'; - colCell.style.paddingTop = '20px'; - colCell.style.paddingLeft = '5%'; - colCell.style.fontStyle = 'italic'; - colCell.style.fontSize = '18px'; - colCell.appendChild(document.createTextNode("Data Domain")); - domain_header_row.appendChild(colCell); - table.appendChild(domain_header_row); - - // Then iteratively add domain details rows - {% for cont_rest in dataset.data_domain.continuous %} - buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); - {% endfor %} - {% for disc_rest in dataset.data_domain.discrete %} - buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); - {% endfor %} - - return table; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "dataset", "name": dataset_name}, + success: function(response) { + ajaxBuildDatasetDetailsTable(response["dataset"], table); + }, + error: function(response) { + alert('Received an error getting serialized ' + dataset_name + ' dataset'); + console.log(response); } - {% endfor %} + }); + return table; } function displayDatasetDetails(name) { @@ -260,9 +417,14 @@ background-color: palegreen; } - #standard-vars { + #dataset-create-form-universal-vars-div { line-height: 25px; - max-width: 300px; + max-width: 350px; + } + + #dataset-create-form-dynamic-vars-div { + line-height: 25px; + max-width: 350px; } .disabled { @@ -314,7 +476,7 @@ - +