Merge pull request #47 from eEcoLiDAR/development

Development

.github/workflows/docker-image.yml

@@ -0,0 +1,30 @@
+name: docker-image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Tag name
+        id: tag_name
+        run: echo ::set-output name=RELEASE_TAG::${GITHUB_REF/refs\/tags\//}
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:${{ steps.tag_name.outputs.RELEASE_TAG }},${{ secrets.DOCKERHUB_USERNAME }}/laserfarm:latest

.github/workflows/test.yaml

@@ -8,7 +8,11 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ]
 
     steps:
     - uses: actions/checkout@v2

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ##[Unreleased]
 
+## [0.1.5] - 2021-11-01
+### Fixed:
+- Compatibility issues (mainly path and logging related) to run on Windows
+
 ## [0.1.4] - 2021-06-29
 ### Fixed:
 - Token authentication can be now used for WebDAV remote data access

CITATION.cff

@@ -23,7 +23,7 @@ authors:
     family-names: Koma
     given-names: Zsófia
 cff-version: "1.0.3"
-date-released: 2021-06-29
+date-released: 2021-11-01
 doi: "10.5281/zenodo.3842780"
 keywords:
   - "airborne laser scanning"
@@ -32,5 +32,5 @@ keywords:
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 title: "Laserfarm: Laserchicken Framework for Applications in Research in Macro-ecology"
-version: "0.1.4"
+version: "0.1.5"
 ...

Dockerfile

@@ -0,0 +1,16 @@
+FROM continuumio/miniconda3 AS build
+
+COPY . /laserfarm
+
+WORKDIR /laserfarm
+RUN apt-get update --allow-releaseinfo-change && apt-get -y install gcc g++
+RUN conda env create -f environment.yml
+RUN conda install -c conda-forge conda-pack
+RUN conda-pack -n laserfarm -o /tmp/env.tar && \
+    mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
+    rm /tmp/env.tar
+RUN /venv/bin/conda-unpack
+
+FROM debian:buster AS runtime
+COPY --from=build /venv /venv
+ENV PATH=/venv/bin:${PATH}

environment.yml

@@ -11,5 +11,5 @@ dependencies:
   - gdal
   - pip
   - pip:
-      - -r file:requirements.txt
+      - -r requirements.txt
       - .

laserfarm/__version__.py

@@ -1 +1 @@
-__version__ = '0.1.4'
+__version__ = '0.1.5'

laserfarm/data_processing.py

@@ -356,7 +356,7 @@ def _get_output_file_dict(path,
         if features and not multi_band_files:
             files = {}
             for feature in features:
-                sub_path = p / feature
+                sub_path = p / _remove_illegal_chars(feature)
                 check_dir_exists(sub_path, should_exist=True, mkdir=True)
                 file_path = (sub_path / file_handle).with_suffix(format)
                 files.update({file_path.as_posix(): [feature]})
@@ -378,3 +378,10 @@ def _get_output_file_dict(path,
         for file in files.keys():
             check_file_exists(file, should_exist=False)
     return files
+
+
+def _remove_illegal_chars(string):
+    # file paths on Windows cannot contain the following characters
+    for char in [">", "<", "/", ":" '"', "\\", "|", "?", "*"]:
+        string = string.replace(char, "")
+    return string

laserfarm/logger.py

@@ -56,17 +56,16 @@ def update_handlers(self):
 
     def remove_handlers(self, stream=False, file=False):
         """ Remove handler instances. """
-        mask = [True for h in self.logger.handlers]
-        for n, handler in enumerate(self.logger.handlers):
+        handlers = self.logger.handlers[:]
+        for handler in handlers:
             if isinstance(handler, logging.StreamHandler) and stream:
-                mask[n] = False
+                self.logger.removeHandler(handler)
             if isinstance(handler, logging.FileHandler) and file:
-                mask[n] = False
                 logger.debug('Terminating stream to logfile: '
                              '{}'.format(handler.baseFilename))
+                self.logger.removeHandler(handler)
                 self._redirect_std_streams(False)
-        self.logger.handlers = [h for n, h in enumerate(self.logger.handlers)
-                                if mask[n]]
+        logging.shutdown()
 
     def _redirect_std_streams(self, redirect):
         if redirect:

laserfarm/pipeline_remote_data.py

@@ -42,9 +42,13 @@ def setup_local_fs(self, input_folder=None, output_folder=None,
             output_folder = tmp_path / '_'.join([self.label, 'output'])
         check_dir_exists(output_folder, should_exist=True, mkdir=True)
         self.output_folder = output_folder
-        logger.info('Output dir set to {}'.format(self.output_folder))
+        logger.info(
+            'Output dir set to {}'.format(self.output_folder.as_posix())
+        )
         if self.logger is not None:
-            self.logger.start_log_to_file(directory=self.output_folder.as_posix())
+            self.logger.start_log_to_file(
+                directory=self.output_folder.as_posix()
+            )
         return self
 
     def setup_webdav_client(self, webdav_options):
@@ -62,10 +66,12 @@ def pullremote(self, remote_origin):
         remote_path = pathlib.Path(remote_origin)
         local_path = self.input_path
         if self.input_path.absolute() != self.input_folder.absolute():
-            remote_path = remote_path.joinpath(self.input_path.name)
+            remote_path = remote_path / self.input_path.name
             if self.input_path.suffix:
                 local_path = self.input_folder
-        logger.info('Pulling from WebDAV {} ...'.format(remote_path))
+        logger.info(
+            'Pulling from WebDAV {} ...'.format(remote_path.as_posix())
+        )
         pull_from_remote(self._wdclient,
                          local_path.as_posix(),
                          remote_path.as_posix())

tests/test_data_processing.py

@@ -14,18 +14,26 @@ class TestInitializeDataProcessing(unittest.TestCase):
     def test_initDefault(self):
         dp = DataProcessing()
         self.assertIsInstance(dp.input_path, pathlib.Path)
-        self.assertEqual(dp.input_path.absolute().as_posix(), os.getcwd())
+        self.assertEqual(
+            dp.input_path.absolute().as_posix(),
+            pathlib.Path.cwd().as_posix()
+        )
 
     def test_initRelativePath(self):
         filepath = 'dir/file.dat'
         dp = DataProcessing(input=filepath)
-        self.assertEqual(dp.input_path.absolute().as_posix(),
-                         os.path.join(os.getcwd(), filepath))
+        self.assertEqual(
+            dp.input_path.absolute().as_posix(),
+            (pathlib.Path.cwd() / filepath).as_posix()
+        )
 
     def test_initAbsolutePath(self):
-        filepath = '/dir/file.dat'
+        filepath = pathlib.Path('/dir/file.dat')
         dp = DataProcessing(input=filepath)
-        self.assertEqual(dp.input_path.absolute().as_posix(), filepath)
+        self.assertEqual(
+            dp.input_path.absolute().as_posix(),
+            filepath.absolute().as_posix()
+        )
 
 
 class TestAddCustomFeature(unittest.TestCase):
@@ -452,6 +460,16 @@ def test_setListOfAttributesAndMultipleSingleBandFiles(self):
         self.assertListEqual(['x', 'y', 'z', feature],
                              _get_attributes_in_PLY_file(path))
 
+    def test_exportSingleBandFileWithFeatureWithIllegalCharacters(self):
+        self.pipeline.output_folder = self._test_dir
+        feature = 'x<0'
+        self.pipeline.export_targets(attributes=[feature],
+                                     multi_band_files=False)
+        output_name = os.path.join('x0', self._output_name)
+        output_path = os.path.join(self._test_dir, output_name)
+        self.assertListEqual(['x', 'y', 'z', feature],
+                             _get_attributes_in_PLY_file(output_path))
+
     def test_addExportOptions(self):
         self.pipeline.output_folder = self._test_dir
         self.pipeline.export_targets(is_binary=True)

tests/test_integration.py

@@ -166,7 +166,7 @@ def test_FullPipeline(self):
 
         for feature in self._features:
             # feature-specific target files are present
-            filename = '{}/tile_{}_{}.ply'.format(feature,
+            filename = '{}/tile_{}_{}.ply'.format(feature.replace('<', ''),
                                                   self._tile_index[0],
                                                   self._tile_index[1])
             filepath = os.path.join(self._test_dir, filename)

tests/test_pipeline_remote_data.py

@@ -35,6 +35,7 @@ def test_logfileIsCreated(self):
         self.pipeline.logger.config(filename=self._test_filename,
                                     level='DEBUG')
         self.pipeline.setup_local_fs(self._test_dir, self._test_dir)
+        self.pipeline.logger.terminate()
         self.assertTrue(os.path.isfile(self._test_filepath))
 
     def test_inputDirectoryNonexistent(self):
@@ -81,16 +82,17 @@ def test_noInputPath(self, pull_from_remote):
     def test_withInputPath(self, pull_from_remote):
         client = Client({})
         input_folder = pathlib.Path(self._test_dir)
-        remote_origin = '/path/to/remote'
+        remote_origin = pathlib.Path('/path/to/remote')
         input_path = self.pipeline.input_folder.joinpath(self._test_filename)
         self.pipeline._wdclient = client
         self.pipeline.input_folder = input_folder
         self.pipeline.input_path = input_path
         self.pipeline.pullremote('/path/to/remote')
-        pull_from_remote.assert_called_once_with(client,
-                                                 input_folder.as_posix(),
-                                                 os.path.join(remote_origin,
-                                                              input_path.name))
+        pull_from_remote.assert_called_once_with(
+            client,
+            input_folder.as_posix(),
+            (remote_origin/self._test_filename).as_posix()
+        )
 
     def test_webdavClientNotSet(self):
         remote_origin = '/path/to/remote'

tests/test_utils.py

@@ -94,12 +94,16 @@ def test_configFromJSON(self):
 
 class TestShellExecuteCmd(unittest.TestCase):
     def test_onlyStdout(self):
-        res = shell_execute_cmd("echo $PWD")
-        self.assertTupleEqual(res, (0, '{}\n\n'.format(os.getcwd())))
+        s = "HelloWorld"
+        res = shell_execute_cmd(f"echo {s}")
+        self.assertEqual(res[0], 0)
+        self.assertEqual(res[1].split()[0], s)
 
     def test_onlyStderr(self):
-        res = shell_execute_cmd("echo $PWD 1>&2")
-        self.assertTupleEqual(res, (0, '\n{}\n'.format(os.getcwd())))
+        s = "HelloWorld"
+        res = shell_execute_cmd(f"echo {s} 1>&2")
+        self.assertEqual(res[0], 0)
+        self.assertEqual(res[1].split()[0], s)
 
     def test_nonzeroReturncode(self):
         res = shell_execute_cmd("exit 1")

tests/tools.py

@@ -105,10 +105,11 @@ def create_test_point_cloud(nx_values=10, grid_spacing=1., offset=0., log=True):
     z = np.random.uniform(-0.5, 0.5, x.size)
     feature_1 = np.zeros_like(x, dtype='int32')
     feature_2 = np.full_like(x, np.nan)
+    feature_3 = np.full_like(x, 0.)
 
     point_cloud = {'vertex': {}}
-    for name, array in zip(['x', 'y', 'z', 'feature_1', 'feature_2'],
-                           [x, y, z, feature_1, feature_2]):
+    for name, array in zip(['x', 'y', 'z', 'feature_1', 'feature_2', 'x<0'],
+                           [x, y, z, feature_1, feature_2, feature_3]):
         point_cloud['vertex'][name] = {'data': array,
                                        'type': array.dtype.name}
     if log:

tutorial.ipynb

@@ -204,6 +204,7 @@
     "    'generate_targets': {\n",
     "        'tile_mesh_size' : 10.0,\n",
     "        'validate' : True,\n",
+    "        'validate_precision': 1.e-3,\n",
     "        **grid\n",
     "    },\n",
     "    'extract_features': {\n",
@@ -431,7 +432,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -445,7 +446,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.8.12"
   }
  },
  "nbformat": 4,