Merge pull request #27 from 12rambau/master

handle bigger files
sepal-contrib · Apr 8, 2021 · 3a24684 · 3a24684
2 parents 4f64eb4 + ab599b9
commit 3a24684
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 45 deletions.
diff --git a/component/scripts/pre_processing.py b/component/scripts/pre_processing.py
@@ -61,10 +61,22 @@ def set_byte_map(class_list, raster, process, output):
         if class_list == []:
             data = raw_data
         else:
-            data = np.zeros_like(raw_data)
+            data = np.zeros_like(raw_data, dtype = np.uint8)
+            total_class = sum([len(c) for c in class_list]) 
+            c = 0
+            output.update_progress(0)
             for index, class_ in enumerate(class_list):
+
+                bool_data = np.zeros_like(raw_data, dtype = np.bool_)
                 for val in class_:
-                    data = data + (raw_data == val) * (index + 1)
+                    bool_data = bool_data + (raw_data == val)
+
+                    data_value = (bool_data * (index + 1)).astype(np.uint8)
+                    data = data + data_value
+
+                    # display the advancement
+                    c += 1
+                    output.update_progress(c/total_class)
 
             data = data.astype(out_meta['dtype'])
 
@@ -82,11 +94,16 @@ def unique(raster):
 
     if raster:
         raster = Path(raster)
+    else: 
+        raise Exception("no raster given")
 
-        with rio.open(raster) as src:
-            data = src.read(1)
-            count = np.bincount(data.flatten())
-            features = np.where(count!=0)[0]
-            features = features.tolist()
+    with rio.open(raster) as src:
+
+        data = src.read(1)
+        count = np.bincount(data.flatten())
+        del data
+
+        features = np.where(count!=0)[0]
+        features = features.tolist()
 
     return features
diff --git a/component/scripts/process.py b/component/scripts/process.py
@@ -65,26 +65,29 @@ def run_gwb_process(process, raster, params_list, title, output, offset):
 
     # file in the output directory
     folder = cp.gwb[process]["folder"]
-    if folder == '':
-        out_log = list(out_dir.glob(f'*.log'))
-        out_files = [f for f in out_dir.glob("*.*") if not f.stem == '.log']
-    else:
-        out_log = list(out_dir.glob(f'*.log'))
-        out_files = next(out_dir.glob(f'{raster.stem}_{folder}*/')).glob('*.*') 
-
+    out_log = list(out_dir.glob(f'*.log'))        
 
     # if log is not there, the comutation didn't even started 
     # I let the display in its current state and change the color of the output to red
     if len(out_log) == 0:
         output.type = 'error'
         return []
 
-    # if the log file is the only file then it has crashed
-
     # read the log 
     with open(out_log[0]) as f:
         log = f.read()
+        output.add_live_msg(v.Html(tag='pre', class_='info--text d-inline', children=[log]))
 
+    # if the log file is the only file then it has crashed
+    try:
+        if folder == '':
+            out_files = [f for f in out_dir.glob("*.*") if not f.stem == '.log']
+        else:
+            out_files = next(out_dir.glob(f'{raster.stem}_{folder}*/')).glob('*.*') 
+    except:
+        output.add_live_msg(v.Html(tag='pre', class_='error--text d-inline', children=[log]), 'error')
+        return []
+
     # copy the files in the result directory 
     files = []
     for f in out_files:

diff --git a/component/tile/convert_tile.py b/component/tile/convert_tile.py
@@ -20,7 +20,10 @@ def __init__(self, io, nb_class):
             items = None, 
             v_model = None, 
             chips = True, 
-            multiple = True
+            small_chips = True,
+            multiple = True,
+            dense = True,
+            deletable_chips = True
         ) for i in range(len(cp.convert[nb_class]['label']))]
         requirements = sw.Markdown(cm.requirement[nb_class])
 
@@ -51,7 +54,6 @@ def _on_click(self, widget, event, data):
 
         # check variables
         if not self.output.check_input(self.io.file, cm.bin.no_file): return widget.toggle_loading()
-        #if not self.output.check_input(len(self.io.foreground), cm.bin.no_classes): return widget.toggle_loading()
 
         # compute the bin map
         try:
@@ -69,8 +71,6 @@ def _on_click(self, widget, event, data):
 
             self.io.set_bin_map(bin_map)
 
-            # add the bin map to the download btn
-
         except Exception as e:
             self.output.add_live_msg(str(e), 'error')
 

diff --git a/component/tile/gwb_tile.py b/component/tile/gwb_tile.py
@@ -47,8 +47,6 @@ def _on_click(self, widget, event, data):
                 offset = self.io.offset
             )
 
-            # add the files to the download links
-
         except Exception as e:
             self.output.add_live_msg(str(e), 'error')
 

diff --git a/mspa_ui.ipynb b/mspa_ui.ipynb
@@ -2,10 +2,39 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "demanding-recognition",
+   "execution_count": 1,
+   "id": "58bfe612",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "30019769b63345808a52157e5573c5c0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Styles()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9f366cd7a8d144eea9b6e3b687e22c6b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "ResizeTrigger()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "from sepal_ui import sepalwidgets as sw \n",
     "\n",
@@ -16,8 +45,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "ideal-netherlands",
+   "execution_count": 2,
+   "id": "eb4408e2",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -26,8 +55,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "essential-index",
+   "execution_count": 3,
+   "id": "e3f96e39",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -36,8 +65,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "secret-jewelry",
+   "execution_count": 4,
+   "id": "9e74f257",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -46,8 +75,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "described-vancouver",
+   "execution_count": 5,
+   "id": "f614cd75",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,18 +85,33 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "handled-protein",
+   "execution_count": 6,
+   "id": "78dea7b2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6bbf5e7d94df4b09964735095d7967a2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Tile(align_center=True, children=[Card(children=[Html(children=['MSPA: Morphological Spatial Pattern Analysis'…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "mspa_title"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "suburban-glance",
+   "execution_count": 7,
+   "id": "ad9c4b73",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -85,20 +129,52 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "peripheral-division",
-   "metadata": {},
-   "outputs": [],
+   "execution_count": 8,
+   "id": "4a502b6f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1521176610f14bb7ace3b5ce4f3b8f41",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "ConvertByte(align_center=True, children=[Card(children=[Html(children=['Select map classes'], tag='h2'), Flex(…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "mspa_convert"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "coordinated-regular",
+   "execution_count": 9,
+   "id": "f3b26650",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5d277b67c65641c58cbd193bdfb3e4dc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "MspaTile(align_center=True, children=[Card(children=[Html(children=['Run Process'], tag='h2'), Flex(children=[…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "mspa_process"
    ]