From 84a96472f30243655e6a241a0b8762e37f635786 Mon Sep 17 00:00:00 2001 From: Pierrick Rambaud Date: Thu, 8 Apr 2021 11:57:42 +0000 Subject: [PATCH 1/3] optimize memory usage --- component/scripts/pre_processing.py | 20 +++++++++++++------- component/tile/convert_tile.py | 28 ++++++++++++++-------------- mspa_ui.ipynb | 22 ++++++++++++---------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/component/scripts/pre_processing.py b/component/scripts/pre_processing.py index d8b90f6..e659748 100644 --- a/component/scripts/pre_processing.py +++ b/component/scripts/pre_processing.py @@ -61,10 +61,11 @@ def set_byte_map(class_list, raster, process, output): if class_list == []: data = raw_data else: - data = np.zeros_like(raw_data) + data = np.zeros_like(raw_data, dtype = np.uint8) for index, class_ in enumerate(class_list): for val in class_: - data = data + (raw_data == val) * (index + 1) + data_value = ((raw_data == val) * (index + 1)).astype(np.uint8) + data = data + data_value data = data.astype(out_meta['dtype']) @@ -82,11 +83,16 @@ def unique(raster): if raster: raster = Path(raster) + else: + raise Exception("no raster given") - with rio.open(raster) as src: - data = src.read(1) - count = np.bincount(data.flatten()) - features = np.where(count!=0)[0] - features = features.tolist() + with rio.open(raster) as src: + + data = src.read(1) + count = np.bincount(data.flatten()) + del data + + features = np.where(count!=0)[0] + features = features.tolist() return features \ No newline at end of file diff --git a/component/tile/convert_tile.py b/component/tile/convert_tile.py index a2860bf..c3088ee 100644 --- a/component/tile/convert_tile.py +++ b/component/tile/convert_tile.py @@ -54,25 +54,25 @@ def _on_click(self, widget, event, data): #if not self.output.check_input(len(self.io.foreground), cm.bin.no_classes): return widget.toggle_loading() # compute the bin map - try: + #try: - # update byte list - self.io.update_byte_list() + # update byte list + self.io.update_byte_list() - # create a bin map - bin_map = cs.set_byte_map( - self.io.byte_list, - self.io.file, - self.io.process, - self.output - ) + # create a bin map + bin_map = cs.set_byte_map( + self.io.byte_list, + self.io.file, + self.io.process, + self.output + ) - self.io.set_bin_map(bin_map) + self.io.set_bin_map(bin_map) - # add the bin map to the download btn + # add the bin map to the download btn - except Exception as e: - self.output.add_live_msg(str(e), 'error') + #except Exception as e: + # self.output.add_live_msg(str(e), 'error') # release the btn diff --git a/mspa_ui.ipynb b/mspa_ui.ipynb index 33bf9e7..f3e5039 100644 --- a/mspa_ui.ipynb +++ b/mspa_ui.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": null, - "id": "demanding-recognition", + "id": "7c81f3bf", "metadata": {}, "outputs": [], "source": [ @@ -17,7 +17,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ideal-netherlands", + "id": "96c9be65", "metadata": {}, "outputs": [], "source": [ @@ -27,7 +27,7 @@ { "cell_type": "code", "execution_count": null, - "id": "essential-index", + "id": "c022d8a1", "metadata": {}, "outputs": [], "source": [ @@ -37,7 +37,7 @@ { "cell_type": "code", "execution_count": null, - "id": "secret-jewelry", + "id": "ebd5d131", "metadata": {}, "outputs": [], "source": [ @@ -47,7 +47,7 @@ { "cell_type": "code", "execution_count": null, - "id": "described-vancouver", + "id": "4aae6289", "metadata": {}, "outputs": [], "source": [ @@ -57,7 +57,7 @@ { "cell_type": "code", "execution_count": null, - "id": "handled-protein", + "id": "8eacb9a2", "metadata": {}, "outputs": [], "source": [ @@ -67,7 +67,7 @@ { "cell_type": "code", "execution_count": null, - "id": "suburban-glance", + "id": "64c0e33a", "metadata": {}, "outputs": [], "source": [ @@ -86,8 +86,10 @@ { "cell_type": "code", "execution_count": null, - "id": "peripheral-division", - "metadata": {}, + "id": "a39cde33", + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "mspa_convert" @@ -96,7 +98,7 @@ { "cell_type": "code", "execution_count": null, - "id": "coordinated-regular", + "id": "1b027967", "metadata": {}, "outputs": [], "source": [ From a27eb29c17e6dbbe99ebb657a184b93365969908 Mon Sep 17 00:00:00 2001 From: Pierrick Rambaud Date: Thu, 8 Apr 2021 12:19:36 +0000 Subject: [PATCH 2/3] display information to the end user --- component/scripts/pre_processing.py | 13 ++++++++++++- mspa_ui.ipynb | 18 +++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/component/scripts/pre_processing.py b/component/scripts/pre_processing.py index e659748..ffaf1aa 100644 --- a/component/scripts/pre_processing.py +++ b/component/scripts/pre_processing.py @@ -62,10 +62,21 @@ def set_byte_map(class_list, raster, process, output): data = raw_data else: data = np.zeros_like(raw_data, dtype = np.uint8) + total_class = sum([len(c) for c in class_list]) + c = 0 + output.update_progress(0) for index, class_ in enumerate(class_list): + + bool_data = np.zeros_like(raw_data, dtype = np.bool_) for val in class_: - data_value = ((raw_data == val) * (index + 1)).astype(np.uint8) + bool_data = bool_data + (raw_data == val) + + data_value = (bool_data * (index + 1)).astype(np.uint8) data = data + data_value + + # display the advancement + c += 1 + output.update_progress(c/total_class) data = data.astype(out_meta['dtype']) diff --git a/mspa_ui.ipynb b/mspa_ui.ipynb index f3e5039..a2e4472 100644 --- a/mspa_ui.ipynb +++ b/mspa_ui.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7c81f3bf", + "id": "1403bcbb", "metadata": {}, "outputs": [], "source": [ @@ -17,7 +17,7 @@ { "cell_type": "code", "execution_count": null, - "id": "96c9be65", + "id": "17734e0a", "metadata": {}, "outputs": [], "source": [ @@ -27,7 +27,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c022d8a1", + "id": "dfcf7083", "metadata": {}, "outputs": [], "source": [ @@ -37,7 +37,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ebd5d131", + "id": "60a47937", "metadata": {}, "outputs": [], "source": [ @@ -47,7 +47,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4aae6289", + "id": "b030d353", "metadata": {}, "outputs": [], "source": [ @@ -57,7 +57,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8eacb9a2", + "id": "0cef413f", "metadata": {}, "outputs": [], "source": [ @@ -67,7 +67,7 @@ { "cell_type": "code", "execution_count": null, - "id": "64c0e33a", + "id": "4e7677fb", "metadata": {}, "outputs": [], "source": [ @@ -86,7 +86,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a39cde33", + "id": "5ed1abb7", "metadata": { "tags": [] }, @@ -98,7 +98,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1b027967", + "id": "2eeea4e6", "metadata": {}, "outputs": [], "source": [ From 4b5ed0f5d18e9a862208e30038f7532eda6deb68 Mon Sep 17 00:00:00 2001 From: Pierrick Rambaud Date: Thu, 8 Apr 2021 12:55:34 +0000 Subject: [PATCH 3/3] catch error when no file is created --- component/scripts/process.py | 21 +++--- component/tile/convert_tile.py | 34 +++++----- component/tile/gwb_tile.py | 2 - mspa_ui.ipynb | 118 +++++++++++++++++++++++++++------ 4 files changed, 125 insertions(+), 50 deletions(-) diff --git a/component/scripts/process.py b/component/scripts/process.py index 225b739..e47fe81 100644 --- a/component/scripts/process.py +++ b/component/scripts/process.py @@ -65,13 +65,7 @@ def run_gwb_process(process, raster, params_list, title, output, offset): # file in the output directory folder = cp.gwb[process]["folder"] - if folder == '': - out_log = list(out_dir.glob(f'*.log')) - out_files = [f for f in out_dir.glob("*.*") if not f.stem == '.log'] - else: - out_log = list(out_dir.glob(f'*.log')) - out_files = next(out_dir.glob(f'{raster.stem}_{folder}*/')).glob('*.*') - + out_log = list(out_dir.glob(f'*.log')) # if log is not there, the comutation didn't even started # I let the display in its current state and change the color of the output to red @@ -79,12 +73,21 @@ def run_gwb_process(process, raster, params_list, title, output, offset): output.type = 'error' return [] - # if the log file is the only file then it has crashed - # read the log with open(out_log[0]) as f: log = f.read() + output.add_live_msg(v.Html(tag='pre', class_='info--text d-inline', children=[log])) + # if the log file is the only file then it has crashed + try: + if folder == '': + out_files = [f for f in out_dir.glob("*.*") if not f.stem == '.log'] + else: + out_files = next(out_dir.glob(f'{raster.stem}_{folder}*/')).glob('*.*') + except: + output.add_live_msg(v.Html(tag='pre', class_='error--text d-inline', children=[log]), 'error') + return [] + # copy the files in the result directory files = [] for f in out_files: diff --git a/component/tile/convert_tile.py b/component/tile/convert_tile.py index c3088ee..b6e85d8 100644 --- a/component/tile/convert_tile.py +++ b/component/tile/convert_tile.py @@ -20,7 +20,10 @@ def __init__(self, io, nb_class): items = None, v_model = None, chips = True, - multiple = True + small_chips = True, + multiple = True, + dense = True, + deletable_chips = True ) for i in range(len(cp.convert[nb_class]['label']))] requirements = sw.Markdown(cm.requirement[nb_class]) @@ -51,28 +54,25 @@ def _on_click(self, widget, event, data): # check variables if not self.output.check_input(self.io.file, cm.bin.no_file): return widget.toggle_loading() - #if not self.output.check_input(len(self.io.foreground), cm.bin.no_classes): return widget.toggle_loading() # compute the bin map - #try: + try: - # update byte list - self.io.update_byte_list() + # update byte list + self.io.update_byte_list() - # create a bin map - bin_map = cs.set_byte_map( - self.io.byte_list, - self.io.file, - self.io.process, - self.output - ) - - self.io.set_bin_map(bin_map) + # create a bin map + bin_map = cs.set_byte_map( + self.io.byte_list, + self.io.file, + self.io.process, + self.output + ) - # add the bin map to the download btn + self.io.set_bin_map(bin_map) - #except Exception as e: - # self.output.add_live_msg(str(e), 'error') + except Exception as e: + self.output.add_live_msg(str(e), 'error') # release the btn diff --git a/component/tile/gwb_tile.py b/component/tile/gwb_tile.py index 3cf4a6c..e439295 100644 --- a/component/tile/gwb_tile.py +++ b/component/tile/gwb_tile.py @@ -47,8 +47,6 @@ def _on_click(self, widget, event, data): offset = self.io.offset ) - # add the files to the download links - except Exception as e: self.output.add_live_msg(str(e), 'error') diff --git a/mspa_ui.ipynb b/mspa_ui.ipynb index a2e4472..aa3c477 100644 --- a/mspa_ui.ipynb +++ b/mspa_ui.ipynb @@ -2,10 +2,39 @@ "cells": [ { "cell_type": "code", - "execution_count": null, - "id": "1403bcbb", + "execution_count": 1, + "id": "58bfe612", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "30019769b63345808a52157e5573c5c0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Styles()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9f366cd7a8d144eea9b6e3b687e22c6b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "ResizeTrigger()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "from sepal_ui import sepalwidgets as sw \n", "\n", @@ -16,8 +45,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "17734e0a", + "execution_count": 2, + "id": "eb4408e2", "metadata": {}, "outputs": [], "source": [ @@ -26,8 +55,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "dfcf7083", + "execution_count": 3, + "id": "e3f96e39", "metadata": {}, "outputs": [], "source": [ @@ -36,8 +65,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "60a47937", + "execution_count": 4, + "id": "9e74f257", "metadata": {}, "outputs": [], "source": [ @@ -46,8 +75,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b030d353", + "execution_count": 5, + "id": "f614cd75", "metadata": {}, "outputs": [], "source": [ @@ -56,18 +85,33 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "0cef413f", + "execution_count": 6, + "id": "78dea7b2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6bbf5e7d94df4b09964735095d7967a2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Tile(align_center=True, children=[Card(children=[Html(children=['MSPA: Morphological Spatial Pattern Analysis'…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "mspa_title" ] }, { "cell_type": "code", - "execution_count": null, - "id": "4e7677fb", + "execution_count": 7, + "id": "ad9c4b73", "metadata": {}, "outputs": [], "source": [ @@ -85,22 +129,52 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "5ed1abb7", + "execution_count": 8, + "id": "4a502b6f", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1521176610f14bb7ace3b5ce4f3b8f41", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "ConvertByte(align_center=True, children=[Card(children=[Html(children=['Select map classes'], tag='h2'), Flex(…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "mspa_convert" ] }, { "cell_type": "code", - "execution_count": null, - "id": "2eeea4e6", + "execution_count": 9, + "id": "f3b26650", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d277b67c65641c58cbd193bdfb3e4dc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "MspaTile(align_center=True, children=[Card(children=[Html(children=['Run Process'], tag='h2'), Flex(children=[…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "mspa_process" ]