Dynamic Dataset creation form

docker/main/dataservice/entrypoint.sh

@@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then
         for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do
             if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then
                 pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}')
-                pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-)
+                pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-)
             fi
         done
         #pip install ${UPDATED_PACKAGES_DIR}/*.whl

docker/nwm_gui/docker-compose.yml

@@ -34,6 +34,7 @@ services:
       args:
         docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)}
         comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?}
+        client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?}
     networks:
       - request-listener-net
     # Call this when starting the container
@@ -62,6 +63,8 @@ services:
     volumes:
       - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static
       - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl
+      # Needed only for speeding debugging
+      #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS
       #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah}
     # Expose Django's port to the internal network so that the web server may access it
     expose:

example.env

@@ -101,6 +101,11 @@ NGEN_BRANCH=master
 ## Python Packages Settings                                           ##
 ########################################################################
 
+## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip)
+PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client
+## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line)
+PYTHON_PACKAGE_NAME_CLIENT=dmod.client
+
 ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip)
 PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication
 ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line)

python/gui/MaaS/cbv/AbstractDatasetView.py

@@ -0,0 +1,29 @@
+from abc import ABC
+from django.views.generic.base import View
+from dmod.client.request_clients import DatasetExternalClient
+import logging
+logger = logging.getLogger("gui_log")
+from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR
+from typing import Dict
+
+
+class AbstractDatasetView(View, DMODMixin, ABC):
+
+    def __init__(self, *args, **kwargs):
+        super(AbstractDatasetView, self).__init__(*args, **kwargs)
+        self._dataset_client = None
+
+    async def get_dataset(self, dataset_name: str) -> Dict[str, dict]:
+        serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name)
+        return serial_dataset
+
+    async def get_datasets(self) -> Dict[str, dict]:
+        serial_datasets = await self.dataset_client.get_serialized_datasets()
+        return serial_datasets
+
+    @property
+    def dataset_client(self) -> DatasetExternalClient:
+        if self._dataset_client is None:
+            self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri,
+                                                         ssl_directory=GUI_STATIC_SSL_DIR)
+        return self._dataset_client

python/gui/MaaS/cbv/DMODProxy.py

@@ -16,6 +16,8 @@
 from pathlib import Path
 from typing import List, Optional, Tuple, Type
 
+GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl')
+
 
 class RequestFormProcessor(ABC):
 
@@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient):
     def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None):
         if ssl_dir is None:
             ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl')
-            ssl_dir = Path('/usr/maas_portal/ssl') #Fixme
+            ssl_dir = GUI_STATIC_SSL_DIR #Fixme
         return ssl_dir
 
     def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None):
@@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) ->
         client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request)
         if event_type == MessageEventType.MODEL_EXEC_REQUEST:
             form_processor_type = ModelExecRequestFormProcessor
+        # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading)
         else:
             raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type)))
 

python/gui/MaaS/cbv/DatasetApiView.py

@@ -0,0 +1,62 @@
+import asyncio
+from django.http import JsonResponse
+from wsgiref.util import FileWrapper
+from django.http.response import StreamingHttpResponse
+from .AbstractDatasetView import AbstractDatasetView
+from .DatasetFileWebsocketFilelike import DatasetFileWebsocketFilelike
+import logging
+logger = logging.getLogger("gui_log")
+
+
+class DatasetApiView(AbstractDatasetView):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _get_dataset_content_details(self, dataset_name: str):
+        result = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_dataset_content_details(name=dataset_name))
+        logger.info(result)
+        return JsonResponse({"contents": result}, status=200)
+
+    def _delete_dataset(self, dataset_name: str) -> JsonResponse:
+        result = asyncio.get_event_loop().run_until_complete(self.dataset_client.delete_dataset(name=dataset_name))
+        return JsonResponse({"successful": result}, status=200)
+
+    def _get_datasets_json(self) -> JsonResponse:
+        serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets())
+        return JsonResponse({"datasets": serial_dataset_map}, status=200)
+
+    def _get_dataset_json(self, dataset_name: str) -> JsonResponse:
+        serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name))
+        return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200)
+
+    def _get_download(self, request, *args, **kwargs):
+        dataset_name = request.GET.get("dataset_name", None)
+        item_name = request.GET.get("item_name", None)
+        chunk_size = 8192
+
+        custom_filelike = DatasetFileWebsocketFilelike(self.dataset_client, dataset_name, item_name)
+
+        response = StreamingHttpResponse(
+            FileWrapper(custom_filelike, chunk_size),
+            content_type="application/octet-stream"
+        )
+        response['Content-Length'] = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_item_size(dataset_name, item_name))
+        response['Content-Disposition'] = "attachment; filename=%s" % item_name
+        return response
+
+    def get(self, request, *args, **kwargs):
+        request_type = request.GET.get("request_type", None)
+        if request_type == 'download_file':
+            return self._get_download(request)
+        elif request_type == 'datasets':
+            return self._get_datasets_json()
+        elif request_type == 'dataset':
+            return self._get_dataset_json(dataset_name=request.GET.get("name", None))
+        elif request_type == 'contents':
+            return self._get_dataset_content_details(dataset_name=request.GET.get("name", None))
+        if request_type == 'delete':
+            return self._delete_dataset(dataset_name=request.GET.get("name", None))
+
+        # TODO: finish
+        return JsonResponse({}, status=400)

python/gui/MaaS/cbv/DatasetFileWebsocketFilelike.py

@@ -0,0 +1,20 @@
+import asyncio
+from typing import AnyStr
+from dmod.client.request_clients import DatasetExternalClient
+
+
+class DatasetFileWebsocketFilelike:
+
+    def __init__(self, client: DatasetExternalClient, dataset_name: str, file_name: str):
+        self._client = client
+        self._dataset_name = dataset_name
+        self._file_name = file_name
+        self._read_index: int = 0
+
+    def read(self, blksize: int) -> AnyStr:
+
+        result = asyncio.get_event_loop().run_until_complete(
+            self._client.download_item_block(dataset_name=self._dataset_name, item_name=self._file_name,
+                                             blk_start=self._read_index, blk_size=blksize))
+        self._read_index += blksize
+        return result