From 8f357cca1a369fa6f95d3bfe7fa5b1fabde15f21 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Fri, 15 Oct 2021 14:04:41 +0200 Subject: [PATCH 01/20] add tests on windows (and macos) --- .github/workflows/test.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 731aecf..fe6d589 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,7 +8,11 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] steps: - uses: actions/checkout@v2 From c2ad93756f13236b5826ab7b58b2ce4bb5f142be Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Fri, 15 Oct 2021 14:11:40 +0200 Subject: [PATCH 02/20] remove joinpath --- laserfarm/pipeline_remote_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laserfarm/pipeline_remote_data.py b/laserfarm/pipeline_remote_data.py index 000aa77..a96fe1a 100644 --- a/laserfarm/pipeline_remote_data.py +++ b/laserfarm/pipeline_remote_data.py @@ -62,7 +62,7 @@ def pullremote(self, remote_origin): remote_path = pathlib.Path(remote_origin) local_path = self.input_path if self.input_path.absolute() != self.input_folder.absolute(): - remote_path = remote_path.joinpath(self.input_path.name) + remote_path = remote_path / self.input_path.name if self.input_path.suffix: local_path = self.input_folder logger.info('Pulling from WebDAV {} ...'.format(remote_path)) From e1bd7c0d1d52cacfaae187a9e6989dd597a9012b Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Wed, 27 Oct 2021 17:07:28 +0200 Subject: [PATCH 03/20] close logging files --- laserfarm/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/laserfarm/logger.py b/laserfarm/logger.py index aac00a6..02c2945 100644 --- a/laserfarm/logger.py +++ b/laserfarm/logger.py @@ -60,10 +60,12 @@ def remove_handlers(self, stream=False, file=False): for n, handler in enumerate(self.logger.handlers): if isinstance(handler, logging.StreamHandler) and stream: mask[n] = False + handler.close() if isinstance(handler, logging.FileHandler) and file: mask[n] = False logger.debug('Terminating stream to logfile: ' '{}'.format(handler.baseFilename)) + handler.close() self._redirect_std_streams(False) self.logger.handlers = [h for n, h in enumerate(self.logger.handlers) if mask[n]] From 5571c3e6ca319d0ddb62db3804d83eab750d30e7 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 09:41:42 +0200 Subject: [PATCH 04/20] gracefully remove handlers --- laserfarm/logger.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/laserfarm/logger.py b/laserfarm/logger.py index 02c2945..040831b 100644 --- a/laserfarm/logger.py +++ b/laserfarm/logger.py @@ -56,19 +56,24 @@ def update_handlers(self): def remove_handlers(self, stream=False, file=False): """ Remove handler instances. """ - mask = [True for h in self.logger.handlers] - for n, handler in enumerate(self.logger.handlers): + handlers = self.logger.handlers[:] + for handler in handlers: if isinstance(handler, logging.StreamHandler) and stream: - mask[n] = False + handler.acquire() + handler.flush() handler.close() + handler.release() + self.logger.removeHandler(handler) if isinstance(handler, logging.FileHandler) and file: - mask[n] = False logger.debug('Terminating stream to logfile: ' '{}'.format(handler.baseFilename)) + handler.acquire() + handler.flush() handler.close() + handler.release() + self.logger.removeHandler(handler) self._redirect_std_streams(False) - self.logger.handlers = [h for n, h in enumerate(self.logger.handlers) - if mask[n]] + def _redirect_std_streams(self, redirect): if redirect: From 4aacdb57c866a2e95b32c03e4fa283d88d7038ca Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:01:17 +0200 Subject: [PATCH 05/20] remove characters that are illegal on Windows from output paths --- laserfarm/data_processing.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/laserfarm/data_processing.py b/laserfarm/data_processing.py index 42871cc..40e1361 100644 --- a/laserfarm/data_processing.py +++ b/laserfarm/data_processing.py @@ -356,7 +356,7 @@ def _get_output_file_dict(path, if features and not multi_band_files: files = {} for feature in features: - sub_path = p / feature + sub_path = p / _remove_illegal_chars(feature) check_dir_exists(sub_path, should_exist=True, mkdir=True) file_path = (sub_path / file_handle).with_suffix(format) files.update({file_path.as_posix(): [feature]}) @@ -378,3 +378,10 @@ def _get_output_file_dict(path, for file in files.keys(): check_file_exists(file, should_exist=False) return files + + +def _remove_illegal_chars(string): + # file paths on Windows cannot contain the following characters + for char in [">", "<", "/", ":" '"', "\\", "|", "?", "*"]: + string.replace(char, "") + return string From 398a1b12aaf34cf4a916b142cefcba544e0fef0c Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:12:15 +0200 Subject: [PATCH 06/20] fix tests making sure paths should work on Windows as well --- tests/test_data_processing.py | 18 +++++++++++++----- tests/test_utils.py | 12 ++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/test_data_processing.py b/tests/test_data_processing.py index d75f0df..e75e5f2 100644 --- a/tests/test_data_processing.py +++ b/tests/test_data_processing.py @@ -14,18 +14,26 @@ class TestInitializeDataProcessing(unittest.TestCase): def test_initDefault(self): dp = DataProcessing() self.assertIsInstance(dp.input_path, pathlib.Path) - self.assertEqual(dp.input_path.absolute().as_posix(), os.getcwd()) + self.assertEqual( + dp.input_path.absolute().as_posix(), + pathlib.Path.cwd().as_posix() + ) def test_initRelativePath(self): filepath = 'dir/file.dat' dp = DataProcessing(input=filepath) - self.assertEqual(dp.input_path.absolute().as_posix(), - os.path.join(os.getcwd(), filepath)) + self.assertEqual( + dp.input_path.absolute().as_posix(), + (pathlib.Path.cwd() / filepath).as_posix() + ) def test_initAbsolutePath(self): - filepath = '/dir/file.dat' + filepath = pathlib.Path('/dir/file.dat') dp = DataProcessing(input=filepath) - self.assertEqual(dp.input_path.absolute().as_posix(), filepath) + self.assertEqual( + dp.input_path.absolute().as_posix(), + filepath.as_posix() + ) class TestAddCustomFeature(unittest.TestCase): diff --git a/tests/test_utils.py b/tests/test_utils.py index e34e932..46bd84b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -94,12 +94,16 @@ def test_configFromJSON(self): class TestShellExecuteCmd(unittest.TestCase): def test_onlyStdout(self): - res = shell_execute_cmd("echo $PWD") - self.assertTupleEqual(res, (0, '{}\n\n'.format(os.getcwd()))) + s = "HelloWorld" + res = shell_execute_cmd(f"echo {s}") + self.assertEqual(res[0], 0) + self.assertEqual(res[1].split()[0], s) def test_onlyStderr(self): - res = shell_execute_cmd("echo $PWD 1>&2") - self.assertTupleEqual(res, (0, '\n{}\n'.format(os.getcwd()))) + s = "HelloWorld" + res = shell_execute_cmd(f"echo {s} 1>&2") + self.assertEqual(res[0], 0) + self.assertEqual(res[1].split()[0], s) def test_nonzeroReturncode(self): res = shell_execute_cmd("exit 1") From ec5bef9050eb20e85e8aab68d415449e65c2ef54 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:22:26 +0200 Subject: [PATCH 07/20] fix test --- tests/test_data_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data_processing.py b/tests/test_data_processing.py index e75e5f2..08e0559 100644 --- a/tests/test_data_processing.py +++ b/tests/test_data_processing.py @@ -32,7 +32,7 @@ def test_initAbsolutePath(self): dp = DataProcessing(input=filepath) self.assertEqual( dp.input_path.absolute().as_posix(), - filepath.as_posix() + filepath.absolute().as_posix() ) From c9f8d18a7d90473bcdcc0338e54b360bdc650076 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:22:45 +0200 Subject: [PATCH 08/20] fix print output --- laserfarm/pipeline_remote_data.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/laserfarm/pipeline_remote_data.py b/laserfarm/pipeline_remote_data.py index a96fe1a..cd1cb2b 100644 --- a/laserfarm/pipeline_remote_data.py +++ b/laserfarm/pipeline_remote_data.py @@ -42,9 +42,13 @@ def setup_local_fs(self, input_folder=None, output_folder=None, output_folder = tmp_path / '_'.join([self.label, 'output']) check_dir_exists(output_folder, should_exist=True, mkdir=True) self.output_folder = output_folder - logger.info('Output dir set to {}'.format(self.output_folder)) + logger.info( + 'Output dir set to {}'.format(self.output_folder.as_posix()) + ) if self.logger is not None: - self.logger.start_log_to_file(directory=self.output_folder.as_posix()) + self.logger.start_log_to_file( + directory=self.output_folder.as_posix() + ) return self def setup_webdav_client(self, webdav_options): @@ -65,7 +69,9 @@ def pullremote(self, remote_origin): remote_path = remote_path / self.input_path.name if self.input_path.suffix: local_path = self.input_folder - logger.info('Pulling from WebDAV {} ...'.format(remote_path)) + logger.info( + 'Pulling from WebDAV {} ...'.format(remote_path.as_posix()) + ) pull_from_remote(self._wdclient, local_path.as_posix(), remote_path.as_posix()) From d002d7a2699092e68a6db8aefe0ae6a5f4cd85c0 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:36:38 +0200 Subject: [PATCH 09/20] fix removal of illegal characters from feature names --- laserfarm/data_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laserfarm/data_processing.py b/laserfarm/data_processing.py index 40e1361..4c569e3 100644 --- a/laserfarm/data_processing.py +++ b/laserfarm/data_processing.py @@ -383,5 +383,5 @@ def _get_output_file_dict(path, def _remove_illegal_chars(string): # file paths on Windows cannot contain the following characters for char in [">", "<", "/", ":" '"', "\\", "|", "?", "*"]: - string.replace(char, "") + string = string.replace(char, "") return string From 3516da378f7736b599e399b2021fe9e0ba6e0f83 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 15:37:33 +0200 Subject: [PATCH 10/20] include logging shutdown --- laserfarm/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laserfarm/logger.py b/laserfarm/logger.py index 040831b..d18a6eb 100644 --- a/laserfarm/logger.py +++ b/laserfarm/logger.py @@ -73,7 +73,7 @@ def remove_handlers(self, stream=False, file=False): handler.release() self.logger.removeHandler(handler) self._redirect_std_streams(False) - + logging.shutdown() def _redirect_std_streams(self, redirect): if redirect: From bccb2dcf1d08e26fe7fd2a61da91c14aeea3bc2f Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 16:09:54 +0200 Subject: [PATCH 11/20] fix tests that had names with illegal characters --- tests/test_data_processing.py | 10 + tests/test_integration.py | 2 +- tests/tools.py | 5 +- tutorial.ipynb | 477 ++++++++++++++++++++++++++++++++-- 4 files changed, 475 insertions(+), 19 deletions(-) diff --git a/tests/test_data_processing.py b/tests/test_data_processing.py index 08e0559..c08c525 100644 --- a/tests/test_data_processing.py +++ b/tests/test_data_processing.py @@ -460,6 +460,16 @@ def test_setListOfAttributesAndMultipleSingleBandFiles(self): self.assertListEqual(['x', 'y', 'z', feature], _get_attributes_in_PLY_file(path)) + def test_exportSingleBandFileWithFeatureWithIllegalCharacters(self): + self.pipeline.output_folder = self._test_dir + feature = 'x<0' + self.pipeline.export_targets(attributes=[feature], + multi_band_files=False) + output_name = os.path.join('x0', self._output_name) + output_path = os.path.join(self._test_dir, output_name) + self.assertListEqual(['x', 'y', 'z', feature], + _get_attributes_in_PLY_file(output_path)) + def test_addExportOptions(self): self.pipeline.output_folder = self._test_dir self.pipeline.export_targets(is_binary=True) diff --git a/tests/test_integration.py b/tests/test_integration.py index 41c3ffa..e6de203 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -166,7 +166,7 @@ def test_FullPipeline(self): for feature in self._features: # feature-specific target files are present - filename = '{}/tile_{}_{}.ply'.format(feature, + filename = '{}/tile_{}_{}.ply'.format(feature.replace('<', ''), self._tile_index[0], self._tile_index[1]) filepath = os.path.join(self._test_dir, filename) diff --git a/tests/tools.py b/tests/tools.py index e69dc65..33649e3 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -105,10 +105,11 @@ def create_test_point_cloud(nx_values=10, grid_spacing=1., offset=0., log=True): z = np.random.uniform(-0.5, 0.5, x.size) feature_1 = np.zeros_like(x, dtype='int32') feature_2 = np.full_like(x, np.nan) + feature_3 = np.full_like(x, 0.) point_cloud = {'vertex': {}} - for name, array in zip(['x', 'y', 'z', 'feature_1', 'feature_2'], - [x, y, z, feature_1, feature_2]): + for name, array in zip(['x', 'y', 'z', 'feature_1', 'feature_2', 'x<0'], + [x, y, z, feature_1, feature_2, feature_3]): point_cloud['vertex'][name] = {'data': array, 'type': array.dtype.name} if log: diff --git a/tutorial.ipynb b/tutorial.ipynb index 35a986a..40f8b5e 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -76,9 +76,196 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "
\n", + "

LocalCluster

\n", + "

3f89021c

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Workers: 2\n", + "
\n", + " Total threads: 2\n", + " \n", + " Total memory: 16.00 GiB\n", + "
Status: runningUsing processes: True
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + " \n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-fe810b42-a2d1-47f3-9190-e6bcdcd200c9

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tcp://127.0.0.1:56132\n", + " \n", + " Workers: 2\n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Total threads: 2\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 16.00 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + " \n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 0

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:56140\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:56141/status\n", + " \n", + " Memory: 8.00 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:56136\n", + "
\n", + " Local directory: /var/tmp/dask-worker-space/dask-worker-space/worker-gwb8uqjx\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 1

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:56139\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:56142/status\n", + " \n", + " Memory: 8.00 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:56135\n", + "
\n", + " Local directory: /var/tmp/dask-worker-space/dask-worker-space/worker-0qaj56n8\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
" + ], + "text/plain": [ + "LocalCluster(3f89021c, 'tcp://127.0.0.1:56132', workers=2, threads=2, memory=16.00 GiB)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "cluster = LocalCluster(processes=True, \n", " n_workers=2, \n", @@ -98,7 +285,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +307,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -140,9 +327,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-10-27 14:59:30,657 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp\n", + "2021-10-27 14:59:30,657 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/retiled\n", + "2021-10-27 14:59:30,658 - laserfarm.retiler - INFO - Setting up the target grid\n", + "2021-10-27 14:59:30,658 - laserfarm.retiler - INFO - Splitting file /var/tmp/C_41CZ2.LAZ with PDAL ...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "001 C_41CZ2 finished\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-10-27 15:00:07,243 - laserfarm.retiler - INFO - ... splitting completed.\n", + "2021-10-27 15:00:07,246 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n", + "2021-10-27 15:00:07,247 - laserfarm.retiler - INFO - ... file C_41CZ2_4.LAZ to tile_170_107\n", + "2021-10-27 15:00:07,250 - laserfarm.retiler - INFO - ... file C_41CZ2_5.LAZ to tile_170_108\n", + "2021-10-27 15:00:07,251 - laserfarm.retiler - INFO - ... file C_41CZ2_7.LAZ to tile_171_108\n", + "2021-10-27 15:00:07,252 - laserfarm.retiler - INFO - ... file C_41CZ2_6.LAZ to tile_171_107\n", + "2021-10-27 15:00:07,253 - laserfarm.retiler - INFO - ... file C_41CZ2_2.LAZ to tile_169_107\n", + "2021-10-27 15:00:07,254 - laserfarm.retiler - INFO - ... file C_41CZ2_3.LAZ to tile_169_108\n", + "2021-10-27 15:00:07,255 - laserfarm.retiler - INFO - ... file C_41CZ2_1.LAZ to tile_169_106\n", + "2021-10-27 15:00:07,255 - laserfarm.retiler - INFO - ... redistributing completed.\n", + "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - Validating split ...\n", + "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - ... 30899674 points in parent file\n", + "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - ... 4131430 points in C_41CZ2_2.LAZ\n", + "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 643355 points in C_41CZ2_3.LAZ\n", + "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 3607773 points in C_41CZ2_1.LAZ\n", + "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 6700111 points in C_41CZ2_7.LAZ\n", + "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 6686994 points in C_41CZ2_5.LAZ\n", + "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... 8149583 points in C_41CZ2_6.LAZ\n", + "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... 980428 points in C_41CZ2_4.LAZ\n", + "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... split validation completed.\n" + ] + } + ], "source": [ "retiling_macro = MacroPipeline()\n", "\n", @@ -169,7 +400,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -185,7 +416,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -204,6 +435,7 @@ " 'generate_targets': {\n", " 'tile_mesh_size' : 10.0,\n", " 'validate' : True,\n", + " 'validate_precision': 1.e-3,\n", " **grid\n", " },\n", " 'extract_features': {\n", @@ -227,9 +459,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/var/tmp/retiled/tile_169_107', '/var/tmp/retiled/tile_169_108', '/var/tmp/retiled/tile_169_106', '/var/tmp/retiled/tile_171_108', '/var/tmp/retiled/tile_170_108', '/var/tmp/retiled/tile_171_107', '/var/tmp/retiled/tile_170_107']\n" + ] + } + ], "source": [ "tiles = []\n", "for file_path in file_paths:\n", @@ -253,7 +493,212 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-10-27 15:00:07,315 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:00:07,316 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:00:07,317 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:00:07,317 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_170_107/C_41CZ2_4.LAZ\n", + "2021-10-27 15:00:07,324 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:00:08,141 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:00:08,141 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:00:08,142 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:00:08,143 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_171_107/C_41CZ2_6.LAZ\n", + "2021-10-27 15:00:08,210 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:00:08,859 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:00:08,859 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:00:10,221 - root - INFO - Cylinder size in Bytes: 4286640343.970202\n", + "2021-10-27 15:00:10,222 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:00:10,222 - root - INFO - Start tree creation\n", + "2021-10-27 15:00:10,339 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:00:10,400 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:00:19,749 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:00:19,749 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:00:21,159 - root - INFO - Cylinder size in Bytes: 5430180069.876887\n", + "2021-10-27 15:00:21,159 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:00:21,159 - root - INFO - Start tree creation\n", + "2021-10-27 15:00:22,329 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:00:22,411 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:00:24,295 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:00:24,296 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:00:24,296 - laserfarm.data_processing - INFO - Checking whether points belong to cell (170,107)\n", + "2021-10-27 15:00:24,323 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:00:24,325 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:00:24,325 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:00:24,325 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:00:26,381 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:00:26,452 - root - ERROR - /opt/miniconda3/envs/test/lib/python3.8/site-packages/laserchicken/feature_extractor/mean_std_coeff_feature_extractor.py:28: RuntimeWarning: invalid value encountered in double_scalars coeff_var_z = std_z / mean_z\n", + "2021-10-27 15:00:26,558 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.18 seconds\n", + "2021-10-27 15:00:26,571 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:00:26,571 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:00:26,572 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:00:26,576 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:00:26,576 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_170_107.ply\n", + "2021-10-27 15:00:27,164 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:00:27,164 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:00:27,174 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:00:27,174 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:00:27,174 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:00:27,175 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_170_108/C_41CZ2_5.LAZ\n", + "2021-10-27 15:00:27,226 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:00:35,883 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:00:35,883 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:00:37,296 - root - INFO - Cylinder size in Bytes: 6103234879.981964\n", + "2021-10-27 15:00:37,296 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:00:37,297 - root - INFO - Start tree creation\n", + "2021-10-27 15:00:38,231 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:00:38,319 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:01:03,301 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:01:03,302 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:01:03,302 - laserfarm.data_processing - INFO - Checking whether points belong to cell (171,107)\n", + "2021-10-27 15:01:03,536 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:01:03,539 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:01:03,539 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:01:03,539 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:01:15,510 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:01:15,511 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:01:15,512 - laserfarm.data_processing - INFO - Checking whether points belong to cell (170,108)\n", + "2021-10-27 15:01:15,718 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:01:15,722 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:01:15,722 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:01:15,722 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:01:22,349 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:01:22,398 - root - ERROR - /opt/miniconda3/envs/test/lib/python3.8/site-packages/laserchicken/feature_extractor/mean_std_coeff_feature_extractor.py:28: RuntimeWarning: invalid value encountered in double_scalars coeff_var_z = std_z / mean_z\n", + "2021-10-27 15:01:23,383 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 1.03 seconds\n", + "2021-10-27 15:01:23,532 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:01:23,533 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:01:23,533 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:01:23,546 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:01:23,546 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_171_107.ply\n", + "2021-10-27 15:01:24,136 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:01:24,137 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:01:24,186 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:01:24,186 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:01:24,186 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:01:24,187 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_171_108/C_41CZ2_7.LAZ\n", + "2021-10-27 15:01:24,256 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:01:30,234 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:01:31,079 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.84 seconds\n", + "2021-10-27 15:01:31,210 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:01:31,211 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:01:31,211 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:01:31,212 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:01:31,212 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_170_108.ply\n", + "2021-10-27 15:01:31,787 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:01:31,787 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:01:31,828 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:01:31,828 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:01:31,829 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:01:31,829 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_106/C_41CZ2_1.LAZ\n", + "2021-10-27 15:01:31,856 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:01:32,429 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:01:32,430 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:01:33,161 - root - INFO - Cylinder size in Bytes: 3381192123.5100083\n", + "2021-10-27 15:01:33,161 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:01:33,162 - root - INFO - Start tree creation\n", + "2021-10-27 15:01:33,988 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:01:34,034 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:01:36,076 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:01:36,076 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:01:36,945 - root - INFO - Cylinder size in Bytes: 4073941954.9515586\n", + "2021-10-27 15:01:36,945 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:01:36,946 - root - INFO - Start tree creation\n", + "2021-10-27 15:01:37,411 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:01:37,471 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:01:58,874 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:01:58,875 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:01:58,875 - laserfarm.data_processing - INFO - Checking whether points belong to cell (169,106)\n", + "2021-10-27 15:01:58,990 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:01:58,992 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:01:58,993 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:01:58,993 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:01:58,993 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:01:58,993 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:01:58,993 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:02:04,550 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:02:04,550 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:02:04,551 - laserfarm.data_processing - INFO - Checking whether points belong to cell (171,108)\n", + "2021-10-27 15:02:04,754 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:02:04,757 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:02:04,757 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:02:04,757 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:02:07,368 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:02:07,910 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.54 seconds\n", + "2021-10-27 15:02:07,972 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:02:07,972 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:02:07,972 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:02:07,972 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:02:07,973 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_169_106.ply\n", + "2021-10-27 15:02:08,515 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:02:08,515 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:02:08,541 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:02:08,541 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:02:08,541 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:02:08,541 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_108/C_41CZ2_3.LAZ\n", + "2021-10-27 15:02:08,553 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:02:09,414 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:02:09,414 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:02:09,608 - root - INFO - Cylinder size in Bytes: 546239521.7583858\n", + "2021-10-27 15:02:09,608 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:02:09,608 - root - INFO - Start tree creation\n", + "2021-10-27 15:02:09,707 - root - INFO - Done with env tree creation\n", + "2021-10-27 15:02:09,717 - root - INFO - Done with target tree creation\n", + "2021-10-27 15:02:14,017 - laserfarm.data_processing - INFO - ... normalization completed.\n", + "2021-10-27 15:02:14,018 - laserfarm.data_processing - INFO - Setting up the target grid\n", + "2021-10-27 15:02:14,018 - laserfarm.data_processing - INFO - Checking whether points belong to cell (169,108)\n", + "2021-10-27 15:02:14,037 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", + "2021-10-27 15:02:14,039 - laserfarm.data_processing - INFO - Building volume of type cell\n", + "2021-10-27 15:02:14,040 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", + "2021-10-27 15:02:14,040 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", + "2021-10-27 15:02:14,040 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", + "2021-10-27 15:02:14,040 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:02:14,040 - root - INFO - Number of points: 13145\n", + "2021-10-27 15:02:15,435 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:02:15,647 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.21 seconds\n", + "2021-10-27 15:02:15,657 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:02:15,658 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:02:15,658 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:02:15,658 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:02:15,659 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_169_108.ply\n", + "2021-10-27 15:02:16,329 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:02:16,329 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:02:16,565 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", + "2021-10-27 15:02:16,566 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", + "2021-10-27 15:02:16,566 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", + "2021-10-27 15:02:16,566 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_107/C_41CZ2_2.LAZ\n", + "2021-10-27 15:02:16,613 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", + "2021-10-27 15:02:20,600 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", + "2021-10-27 15:02:21,527 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.93 seconds\n", + "2021-10-27 15:02:21,658 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", + "2021-10-27 15:02:21,659 - root - INFO - ['coeff_var_normalized_height']\n", + "2021-10-27 15:02:21,659 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", + "2021-10-27 15:02:21,661 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", + "2021-10-27 15:02:21,661 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_171_108.ply\n", + "2021-10-27 15:02:22,179 - laserfarm.data_processing - INFO - ... loading completed.\n", + "2021-10-27 15:02:22,180 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", + "2021-10-27 15:02:22,256 - laserfarm.data_processing - INFO - ... exporting completed.\n", + "2021-10-27 15:02:22,256 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", + "2021-10-27 15:02:26,783 - root - INFO - Cylinder size in Bytes: 21659576843.991497\n", + "2021-10-27 15:02:26,783 - root - INFO - Memory size in Bytes: 17179869184\n", + "2021-10-27 15:02:26,783 - root - INFO - Number of points: 1314548\n" + ] + } + ], "source": [ "dp_macro = MacroPipeline()\n", "\n", @@ -431,7 +876,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -445,7 +890,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.12" } }, "nbformat": 4, From edbf5f64d1114dc7a30f1afee6aa579388b0b632 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 16:10:43 +0200 Subject: [PATCH 12/20] clean up logger and fix test --- laserfarm/logger.py | 8 -------- tests/test_pipeline_remote_data.py | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/laserfarm/logger.py b/laserfarm/logger.py index d18a6eb..ac897f7 100644 --- a/laserfarm/logger.py +++ b/laserfarm/logger.py @@ -59,18 +59,10 @@ def remove_handlers(self, stream=False, file=False): handlers = self.logger.handlers[:] for handler in handlers: if isinstance(handler, logging.StreamHandler) and stream: - handler.acquire() - handler.flush() - handler.close() - handler.release() self.logger.removeHandler(handler) if isinstance(handler, logging.FileHandler) and file: logger.debug('Terminating stream to logfile: ' '{}'.format(handler.baseFilename)) - handler.acquire() - handler.flush() - handler.close() - handler.release() self.logger.removeHandler(handler) self._redirect_std_streams(False) logging.shutdown() diff --git a/tests/test_pipeline_remote_data.py b/tests/test_pipeline_remote_data.py index 9dd8a31..c258d2a 100644 --- a/tests/test_pipeline_remote_data.py +++ b/tests/test_pipeline_remote_data.py @@ -35,6 +35,7 @@ def test_logfileIsCreated(self): self.pipeline.logger.config(filename=self._test_filename, level='DEBUG') self.pipeline.setup_local_fs(self._test_dir, self._test_dir) + self.pipeline.logger.terminate() self.assertTrue(os.path.isfile(self._test_filepath)) def test_inputDirectoryNonexistent(self): From 0ded9acf7b5167ae2531515cc8e390f93efa4469 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 16:17:50 +0200 Subject: [PATCH 13/20] fix notebook tutorial --- tutorial.ipynb | 472 ++----------------------------------------------- 1 file changed, 14 insertions(+), 458 deletions(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index 40f8b5e..1c1e3ab 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -76,196 +76,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "
\n", - "
\n", - "
\n", - "

LocalCluster

\n", - "

3f89021c

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "\n", - " \n", - "
\n", - " Dashboard: http://127.0.0.1:8787/status\n", - " \n", - " Workers: 2\n", - "
\n", - " Total threads: 2\n", - " \n", - " Total memory: 16.00 GiB\n", - "
Status: runningUsing processes: True
\n", - "\n", - "
\n", - " \n", - "

Scheduler Info

\n", - " \n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "

Scheduler

\n", - "

Scheduler-fe810b42-a2d1-47f3-9190-e6bcdcd200c9

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Comm: tcp://127.0.0.1:56132\n", - " \n", - " Workers: 2\n", - "
\n", - " Dashboard: http://127.0.0.1:8787/status\n", - " \n", - " Total threads: 2\n", - "
\n", - " Started: Just now\n", - " \n", - " Total memory: 16.00 GiB\n", - "
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "

Workers

\n", - " \n", - "\n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "

Worker: 0

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - "
\n", - " Comm: tcp://127.0.0.1:56140\n", - " \n", - " Total threads: 1\n", - "
\n", - " Dashboard: http://127.0.0.1:56141/status\n", - " \n", - " Memory: 8.00 GiB\n", - "
\n", - " Nanny: tcp://127.0.0.1:56136\n", - "
\n", - " Local directory: /var/tmp/dask-worker-space/dask-worker-space/worker-gwb8uqjx\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "

Worker: 1

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - "
\n", - " Comm: tcp://127.0.0.1:56139\n", - " \n", - " Total threads: 1\n", - "
\n", - " Dashboard: http://127.0.0.1:56142/status\n", - " \n", - " Memory: 8.00 GiB\n", - "
\n", - " Nanny: tcp://127.0.0.1:56135\n", - "
\n", - " Local directory: /var/tmp/dask-worker-space/dask-worker-space/worker-0qaj56n8\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "\n", - "
\n", - "
\n", - "\n", - "
\n", - "
\n", - "
" - ], - "text/plain": [ - "LocalCluster(3f89021c, 'tcp://127.0.0.1:56132', workers=2, threads=2, memory=16.00 GiB)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "cluster = LocalCluster(processes=True, \n", " n_workers=2, \n", @@ -285,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -307,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -327,53 +140,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-10-27 14:59:30,657 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp\n", - "2021-10-27 14:59:30,657 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/retiled\n", - "2021-10-27 14:59:30,658 - laserfarm.retiler - INFO - Setting up the target grid\n", - "2021-10-27 14:59:30,658 - laserfarm.retiler - INFO - Splitting file /var/tmp/C_41CZ2.LAZ with PDAL ...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "001 C_41CZ2 finished\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-10-27 15:00:07,243 - laserfarm.retiler - INFO - ... splitting completed.\n", - "2021-10-27 15:00:07,246 - laserfarm.retiler - INFO - Redistributing files to tiles ...\n", - "2021-10-27 15:00:07,247 - laserfarm.retiler - INFO - ... file C_41CZ2_4.LAZ to tile_170_107\n", - "2021-10-27 15:00:07,250 - laserfarm.retiler - INFO - ... file C_41CZ2_5.LAZ to tile_170_108\n", - "2021-10-27 15:00:07,251 - laserfarm.retiler - INFO - ... file C_41CZ2_7.LAZ to tile_171_108\n", - "2021-10-27 15:00:07,252 - laserfarm.retiler - INFO - ... file C_41CZ2_6.LAZ to tile_171_107\n", - "2021-10-27 15:00:07,253 - laserfarm.retiler - INFO - ... file C_41CZ2_2.LAZ to tile_169_107\n", - "2021-10-27 15:00:07,254 - laserfarm.retiler - INFO - ... file C_41CZ2_3.LAZ to tile_169_108\n", - "2021-10-27 15:00:07,255 - laserfarm.retiler - INFO - ... file C_41CZ2_1.LAZ to tile_169_106\n", - "2021-10-27 15:00:07,255 - laserfarm.retiler - INFO - ... redistributing completed.\n", - "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - Validating split ...\n", - "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - ... 30899674 points in parent file\n", - "2021-10-27 15:00:07,256 - laserfarm.retiler - INFO - ... 4131430 points in C_41CZ2_2.LAZ\n", - "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 643355 points in C_41CZ2_3.LAZ\n", - "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 3607773 points in C_41CZ2_1.LAZ\n", - "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 6700111 points in C_41CZ2_7.LAZ\n", - "2021-10-27 15:00:07,257 - laserfarm.retiler - INFO - ... 6686994 points in C_41CZ2_5.LAZ\n", - "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... 8149583 points in C_41CZ2_6.LAZ\n", - "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... 980428 points in C_41CZ2_4.LAZ\n", - "2021-10-27 15:00:07,258 - laserfarm.retiler - INFO - ... split validation completed.\n" - ] - } - ], + "outputs": [], "source": [ "retiling_macro = MacroPipeline()\n", "\n", @@ -400,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -416,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -459,17 +228,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/var/tmp/retiled/tile_169_107', '/var/tmp/retiled/tile_169_108', '/var/tmp/retiled/tile_169_106', '/var/tmp/retiled/tile_171_108', '/var/tmp/retiled/tile_170_108', '/var/tmp/retiled/tile_171_107', '/var/tmp/retiled/tile_170_107']\n" - ] - } - ], + "outputs": [], "source": [ "tiles = []\n", "for file_path in file_paths:\n", @@ -493,212 +254,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-10-27 15:00:07,315 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:00:07,316 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:00:07,317 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:00:07,317 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_170_107/C_41CZ2_4.LAZ\n", - "2021-10-27 15:00:07,324 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:00:08,141 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:00:08,141 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:00:08,142 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:00:08,143 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_171_107/C_41CZ2_6.LAZ\n", - "2021-10-27 15:00:08,210 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:00:08,859 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:00:08,859 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:00:10,221 - root - INFO - Cylinder size in Bytes: 4286640343.970202\n", - "2021-10-27 15:00:10,222 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:00:10,222 - root - INFO - Start tree creation\n", - "2021-10-27 15:00:10,339 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:00:10,400 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:00:19,749 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:00:19,749 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:00:21,159 - root - INFO - Cylinder size in Bytes: 5430180069.876887\n", - "2021-10-27 15:00:21,159 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:00:21,159 - root - INFO - Start tree creation\n", - "2021-10-27 15:00:22,329 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:00:22,411 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:00:24,295 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:00:24,296 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:00:24,296 - laserfarm.data_processing - INFO - Checking whether points belong to cell (170,107)\n", - "2021-10-27 15:00:24,323 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:00:24,325 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:00:24,325 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:00:24,325 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:00:24,325 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:00:26,381 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:00:26,452 - root - ERROR - /opt/miniconda3/envs/test/lib/python3.8/site-packages/laserchicken/feature_extractor/mean_std_coeff_feature_extractor.py:28: RuntimeWarning: invalid value encountered in double_scalars coeff_var_z = std_z / mean_z\n", - "2021-10-27 15:00:26,558 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.18 seconds\n", - "2021-10-27 15:00:26,571 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:00:26,571 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:00:26,572 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:00:26,576 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:00:26,576 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_170_107.ply\n", - "2021-10-27 15:00:27,164 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:00:27,164 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:00:27,174 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:00:27,174 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:00:27,174 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:00:27,175 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_170_108/C_41CZ2_5.LAZ\n", - "2021-10-27 15:00:27,226 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:00:35,883 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:00:35,883 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:00:37,296 - root - INFO - Cylinder size in Bytes: 6103234879.981964\n", - "2021-10-27 15:00:37,296 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:00:37,297 - root - INFO - Start tree creation\n", - "2021-10-27 15:00:38,231 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:00:38,319 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:01:03,301 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:01:03,302 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:01:03,302 - laserfarm.data_processing - INFO - Checking whether points belong to cell (171,107)\n", - "2021-10-27 15:01:03,536 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:01:03,538 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:01:03,539 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:01:03,539 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:01:03,539 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:01:15,510 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:01:15,511 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:01:15,512 - laserfarm.data_processing - INFO - Checking whether points belong to cell (170,108)\n", - "2021-10-27 15:01:15,718 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:01:15,721 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:01:15,722 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:01:15,722 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:01:15,722 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:01:22,349 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:01:22,398 - root - ERROR - /opt/miniconda3/envs/test/lib/python3.8/site-packages/laserchicken/feature_extractor/mean_std_coeff_feature_extractor.py:28: RuntimeWarning: invalid value encountered in double_scalars coeff_var_z = std_z / mean_z\n", - "2021-10-27 15:01:23,383 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 1.03 seconds\n", - "2021-10-27 15:01:23,532 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:01:23,533 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:01:23,533 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:01:23,546 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:01:23,546 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_171_107.ply\n", - "2021-10-27 15:01:24,136 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:01:24,137 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:01:24,186 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:01:24,186 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:01:24,186 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:01:24,187 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_171_108/C_41CZ2_7.LAZ\n", - "2021-10-27 15:01:24,256 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:01:30,234 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:01:31,079 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.84 seconds\n", - "2021-10-27 15:01:31,210 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:01:31,211 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:01:31,211 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:01:31,212 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:01:31,212 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_170_108.ply\n", - "2021-10-27 15:01:31,787 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:01:31,787 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:01:31,828 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:01:31,828 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:01:31,829 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:01:31,829 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_106/C_41CZ2_1.LAZ\n", - "2021-10-27 15:01:31,856 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:01:32,429 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:01:32,430 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:01:33,161 - root - INFO - Cylinder size in Bytes: 3381192123.5100083\n", - "2021-10-27 15:01:33,161 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:01:33,162 - root - INFO - Start tree creation\n", - "2021-10-27 15:01:33,988 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:01:34,034 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:01:36,076 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:01:36,076 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:01:36,945 - root - INFO - Cylinder size in Bytes: 4073941954.9515586\n", - "2021-10-27 15:01:36,945 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:01:36,946 - root - INFO - Start tree creation\n", - "2021-10-27 15:01:37,411 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:01:37,471 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:01:58,874 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:01:58,875 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:01:58,875 - laserfarm.data_processing - INFO - Checking whether points belong to cell (169,106)\n", - "2021-10-27 15:01:58,990 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:01:58,992 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:01:58,993 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:01:58,993 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:01:58,993 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:01:58,993 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:01:58,993 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:02:04,550 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:02:04,550 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:02:04,551 - laserfarm.data_processing - INFO - Checking whether points belong to cell (171,108)\n", - "2021-10-27 15:02:04,754 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:02:04,756 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:02:04,757 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:02:04,757 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:02:04,757 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:02:07,368 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:02:07,910 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.54 seconds\n", - "2021-10-27 15:02:07,972 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:02:07,972 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:02:07,972 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:02:07,972 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:02:07,973 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_169_106.ply\n", - "2021-10-27 15:02:08,515 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:02:08,515 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:02:08,541 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:02:08,541 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:02:08,541 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:02:08,541 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_108/C_41CZ2_3.LAZ\n", - "2021-10-27 15:02:08,553 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:02:09,414 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:02:09,414 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:02:09,608 - root - INFO - Cylinder size in Bytes: 546239521.7583858\n", - "2021-10-27 15:02:09,608 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:02:09,608 - root - INFO - Start tree creation\n", - "2021-10-27 15:02:09,707 - root - INFO - Done with env tree creation\n", - "2021-10-27 15:02:09,717 - root - INFO - Done with target tree creation\n", - "2021-10-27 15:02:14,017 - laserfarm.data_processing - INFO - ... normalization completed.\n", - "2021-10-27 15:02:14,018 - laserfarm.data_processing - INFO - Setting up the target grid\n", - "2021-10-27 15:02:14,018 - laserfarm.data_processing - INFO - Checking whether points belong to cell (169,108)\n", - "2021-10-27 15:02:14,037 - laserfarm.data_processing - INFO - Generating target point mesh with 10.0m spacing \n", - "2021-10-27 15:02:14,039 - laserfarm.data_processing - INFO - Building volume of type cell\n", - "2021-10-27 15:02:14,040 - laserfarm.data_processing - INFO - Constructing neighborhoods\n", - "2021-10-27 15:02:14,040 - laserfarm.data_processing - INFO - Starting feature extraction ...\n", - "2021-10-27 15:02:14,040 - root - INFO - Cylinder size in Bytes: 26138050877.867085\n", - "2021-10-27 15:02:14,040 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:02:14,040 - root - INFO - Number of points: 13145\n", - "2021-10-27 15:02:15,435 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:02:15,647 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.21 seconds\n", - "2021-10-27 15:02:15,657 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:02:15,658 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:02:15,658 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:02:15,658 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:02:15,659 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_169_108.ply\n", - "2021-10-27 15:02:16,329 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:02:16,329 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:02:16,565 - laserfarm.pipeline_remote_data - INFO - Input dir set to /var/tmp/retiled\n", - "2021-10-27 15:02:16,566 - laserfarm.pipeline_remote_data - INFO - Output dir set to /var/tmp/targets\n", - "2021-10-27 15:02:16,566 - laserfarm.data_processing - INFO - Loading point cloud data ...\n", - "2021-10-27 15:02:16,566 - laserfarm.data_processing - INFO - ... loading /var/tmp/retiled/tile_169_107/C_41CZ2_2.LAZ\n", - "2021-10-27 15:02:16,613 - pylas.lasreader - ERROR - lazrs failed to decompress points: lazrs is not installed\n", - "2021-10-27 15:02:20,600 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\"\n", - "2021-10-27 15:02:21,527 - root - INFO - Extracting feature(s) \"['mean_normalized_height', 'std_normalized_height', 'coeff_var_normalized_height']\" took 0.93 seconds\n", - "2021-10-27 15:02:21,658 - root - INFO - The following unrequested features were calculated as a side effect, but will not be returned:\n", - "2021-10-27 15:02:21,659 - root - INFO - ['coeff_var_normalized_height']\n", - "2021-10-27 15:02:21,659 - laserfarm.data_processing - INFO - ... feature extraction completed.\n", - "2021-10-27 15:02:21,661 - laserfarm.data_processing - INFO - Exporting target point-cloud ...\n", - "2021-10-27 15:02:21,661 - laserfarm.data_processing - INFO - ... exporting /var/tmp/targets/tile_171_108.ply\n", - "2021-10-27 15:02:22,179 - laserfarm.data_processing - INFO - ... loading completed.\n", - "2021-10-27 15:02:22,180 - laserfarm.data_processing - INFO - Normalizing point-cloud heights ...\n", - "2021-10-27 15:02:22,256 - laserfarm.data_processing - INFO - ... exporting completed.\n", - "2021-10-27 15:02:22,256 - laserfarm.data_processing - INFO - Clearing cached KDTrees ...\n", - "2021-10-27 15:02:26,783 - root - INFO - Cylinder size in Bytes: 21659576843.991497\n", - "2021-10-27 15:02:26,783 - root - INFO - Memory size in Bytes: 17179869184\n", - "2021-10-27 15:02:26,783 - root - INFO - Number of points: 1314548\n" - ] - } - ], + "outputs": [], "source": [ "dp_macro = MacroPipeline()\n", "\n", From 4942b46b6b51dfe62cb60045131a534b41f4e13f Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Thu, 28 Oct 2021 16:36:36 +0200 Subject: [PATCH 14/20] fix test --- tests/test_pipeline_remote_data.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline_remote_data.py b/tests/test_pipeline_remote_data.py index c258d2a..9d119cd 100644 --- a/tests/test_pipeline_remote_data.py +++ b/tests/test_pipeline_remote_data.py @@ -82,16 +82,17 @@ def test_noInputPath(self, pull_from_remote): def test_withInputPath(self, pull_from_remote): client = Client({}) input_folder = pathlib.Path(self._test_dir) - remote_origin = '/path/to/remote' + remote_origin = pathlib.Path('/path/to/remote') input_path = self.pipeline.input_folder.joinpath(self._test_filename) self.pipeline._wdclient = client self.pipeline.input_folder = input_folder self.pipeline.input_path = input_path self.pipeline.pullremote('/path/to/remote') - pull_from_remote.assert_called_once_with(client, - input_folder.as_posix(), - os.path.join(remote_origin, - input_path.name)) + pull_from_remote.assert_called_once_with( + client, + input_folder.as_posix(), + (remote_origin/self._test_filename).as_posix() + ) def test_webdavClientNotSet(self): remote_origin = '/path/to/remote' From d1d89d20ef86766e64b7b4f85993b16e5ff67c3f Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 14:55:08 +0100 Subject: [PATCH 15/20] fix test --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2573fc..f149bfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ##[Unreleased] +### Fixed: +- Compatibility issues (mainly path and logging related) to run on Windows ## [0.1.4] - 2021-06-29 ### Fixed: From 482fd905d5a1e6bd6c64a7aa37cc7d8d7054ea99 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 14:56:49 +0100 Subject: [PATCH 16/20] fix conda environment file --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 868fd1d..c152f1a 100644 --- a/environment.yml +++ b/environment.yml @@ -11,5 +11,5 @@ dependencies: - gdal - pip - pip: - - -r file:requirements.txt + - -r requirements.txt - . From 8758f65656aa32e1239ad3bbd0a04dc3e5c52117 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 15:39:24 +0100 Subject: [PATCH 17/20] add Dockerfile --- Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fcd5237 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM continuumio/miniconda3 AS build + +COPY . /laserfarm + +WORKDIR /laserfarm +RUN apt-get update --allow-releaseinfo-change && apt-get -y install gcc g++ +RUN conda env create -f environment.yml +RUN conda install -c conda-forge conda-pack +RUN conda-pack -n laserfarm -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +FROM debian:buster AS runtime +COPY --from=build /venv /venv +ENV PATH=/venv/bin:${PATH} \ No newline at end of file From 191196979f16bcd95e92ef58c2a17ee640582bfa Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 15:48:07 +0100 Subject: [PATCH 18/20] add github action for docker image build&push --- .github/workflows/docker-image.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..59d3a0f --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,27 @@ +name: docker-image + +on: + push: + branches: + - 'docker-image' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:latest \ No newline at end of file From eb8e30a1d4de4ad47f65f9019447f3b707bea3f8 Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 16:49:22 +0100 Subject: [PATCH 19/20] update github action action triggered for releases only --- .github/workflows/docker-image.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 59d3a0f..6713448 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,9 +1,8 @@ name: docker-image on: - push: - branches: - - 'docker-image' + release: + types: [published] jobs: docker: @@ -12,6 +11,10 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - + name: Tag name + id: tag_name + run: echo ::set-output name=RELEASE_TAG::${GITHUB_REF/refs\/tags\//} - name: Login to DockerHub uses: docker/login-action@v1 @@ -24,4 +27,4 @@ jobs: with: context: . push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:latest \ No newline at end of file + tags: ${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:${{ steps.tag_name.outputs.RELEASE_TAG }},${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:latest \ No newline at end of file From bc8531ba4fec5516b1a53e966e1f3943c3ac811e Mon Sep 17 00:00:00 2001 From: Francesco Nattino Date: Mon, 1 Nov 2021 16:55:56 +0100 Subject: [PATCH 20/20] prepare release --- CHANGELOG.md | 2 ++ CITATION.cff | 4 ++-- laserfarm/__version__.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f149bfe..ce058e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ##[Unreleased] + +## [0.1.5] - 2021-11-01 ### Fixed: - Compatibility issues (mainly path and logging related) to run on Windows diff --git a/CITATION.cff b/CITATION.cff index 4e6f1e5..b22d4d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -23,7 +23,7 @@ authors: family-names: Koma given-names: Zsófia cff-version: "1.0.3" -date-released: 2021-06-29 +date-released: 2021-11-01 doi: "10.5281/zenodo.3842780" keywords: - "airborne laser scanning" @@ -32,5 +32,5 @@ keywords: license: "Apache-2.0" message: "If you use this software, please cite it using these metadata." title: "Laserfarm: Laserchicken Framework for Applications in Research in Macro-ecology" -version: "0.1.4" +version: "0.1.5" ... diff --git a/laserfarm/__version__.py b/laserfarm/__version__.py index 7525d19..66a87bb 100644 --- a/laserfarm/__version__.py +++ b/laserfarm/__version__.py @@ -1 +1 @@ -__version__ = '0.1.4' +__version__ = '0.1.5'