[FEAT] install: Download Osmosis and Osmfilter on Windows during first run instead of shipping with the tool

tests/test_downloader.py

@@ -5,15 +5,19 @@
 # import sys
 import unittest
 import time
+import shutil
+import platform
 
 # import custom python packages
 # sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from wahoomc.downloader import older_than_x_days
 from wahoomc.downloader import download_file
 from wahoomc.downloader import get_osm_pbf_filepath_url
+from wahoomc.downloader import download_tooling_win
 from wahoomc.downloader import Downloader
 from wahoomc import constants
+from wahoomc.constants_functions import get_tooling_win_path
 
 
 class TestDownloader(unittest.TestCase):
@@ -149,6 +153,22 @@ def test_check_dl_needed_geofabrik(self):
         self.assertTrue(
             self.o_downloader.should_geofabrik_file_be_downloaded())
 
+    def test_download_windows_files(self):
+        """
+        Test if Windows tooling files download is successful
+        """
+        if platform.system() == "Windows":
+            path = os.path.join(constants.USER_TOOLING_WIN_DIR)
+
+            if os.path.exists(path):
+                shutil.rmtree(path)
+
+            os.makedirs(constants.USER_TOOLING_WIN_DIR, exist_ok=True)
+            download_tooling_win()
+
+            self.assertTrue(
+                os.path.exists(get_tooling_win_path('osmfilter.exe', in_user_dir=True)))
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_setup.py

@@ -32,18 +32,18 @@ def test_installed_programs_mac(self):
 
     def test_installed_programs_windows(self):
         """
-        tests, if the mac-relevant programs are installed
+        tests, if the windows-relevant programs are installed
         """
         if platform.system() == "Windows":
             self.check_installation_of_program("java")
 
             self.assertTrue(os.path.exists(get_tooling_win_path(
-                ['Osmosis', 'bin', 'osmosis.bat'])))
+                os.path.join('Osmosis', 'bin', 'osmosis.bat'), in_user_dir=True)))
             self.assertTrue(os.path.exists(
-                get_tooling_win_path(['osmconvert.exe'])))
+                get_tooling_win_path('osmconvert.exe')))
             self.assertTrue(os.path.exists(
-                get_tooling_win_path(['osmfilter.exe'])))
-            self.assertTrue(os.path.exists(get_tooling_win_path(['7za.exe'])))
+                get_tooling_win_path('osmfilter.exe', in_user_dir=True)))
+            self.assertTrue(os.path.exists(get_tooling_win_path('7za.exe')))
 
     def check_installation_of_program(self, program):
         """

wahoomc/constants.py

@@ -15,6 +15,9 @@
 GEOFABRIK_PATH = os.path.join(USER_DL_DIR, 'geofabrik.json')
 USER_OUTPUT_DIR = os.path.join(USER_WAHOO_MC, '_tiles')
 USER_CONFIG_DIR = os.path.join(USER_WAHOO_MC, '_config')
+USER_TOOLING_WIN_DIR = os.path.join(USER_DL_DIR, 'tooling_win')
+OSMOSIS_WIN_FILE_PATH = os.path.join(
+    USER_TOOLING_WIN_DIR, 'Osmosis', 'bin', 'osmosis.bat')
 
 # Python Package - wahooMapsCreator directory
 WAHOO_MC_DIR = os.path.dirname(__file__)

wahoomc/constants_functions.py

@@ -7,12 +7,14 @@
 import sys
 import logging
 import os
+import struct
 
 # import custom python packages
 from wahoomc import constants
 from wahoomc.constants import RESOURCES_DIR
 from wahoomc.constants import TOOLING_WIN_DIR
 from wahoomc.constants import USER_CONFIG_DIR
+from wahoomc.constants import USER_TOOLING_WIN_DIR
 from wahoomc.file_directory_functions import read_json_file_generic
 
 log = logging.getLogger('main-logger')
@@ -178,11 +180,25 @@ def transl_tag_value(sys_platform, separator, tag, value):
     return to_append
 
 
-def get_tooling_win_path(path_in_tooling_win):
+def get_tooling_win_path(path_in_tooling_win, in_user_dir=False):
     """
     return path to a tooling in the tooling_win directory and the given path
+    OR from the user tooling_win directory
     """
-    return os.path.join(TOOLING_WIN_DIR, *path_in_tooling_win)
+    if in_user_dir:
+        tooling_dir = USER_TOOLING_WIN_DIR
+    else:
+        tooling_dir = TOOLING_WIN_DIR
+
+    # special for osmconvert: handle 32 and 64 bit here
+    if path_in_tooling_win in ('osmconvert', 'osmconvert.exe'):
+        if 8 * struct.calcsize("P") == 32:
+            return os.path.join(tooling_dir, path_in_tooling_win)
+        # 64 bit: replace with 64 in the end
+        return os.path.join(tooling_dir, path_in_tooling_win.replace("osmconvert", "osmconvert64-0.8.8p"))
+
+    # all other "toolings": concatenate with win tooling dir
+    return os.path.join(tooling_dir, path_in_tooling_win)
 
 
 def get_tag_wahoo_xml_path(tag_wahoo_xml):

wahoomc/downloader.py

@@ -10,16 +10,19 @@
 import sys
 import time
 import logging
+import platform
 
 # import custom python packages
 from wahoomc.file_directory_functions import download_url_to_file, unzip
 from wahoomc.constants_functions import translate_country_input_to_geofabrik, \
-    get_geofabrik_region_of_country
+    get_geofabrik_region_of_country, get_tooling_win_path
 
 from wahoomc.constants import USER_DL_DIR
 from wahoomc.constants import USER_MAPS_DIR
 from wahoomc.constants import LAND_POLYGONS_PATH
 from wahoomc.constants import GEOFABRIK_PATH
+from wahoomc.constants import OSMOSIS_WIN_FILE_PATH
+from wahoomc.constants import USER_TOOLING_WIN_DIR
 
 log = logging.getLogger('main-logger')
 
@@ -33,22 +36,29 @@ def older_than_x_days(file_creation_timestamp, max_days_old):
     return bool(file_creation_timestamp < to_old_timestamp)
 
 
-def download_file(target_filepath, url, is_zip):
+def download_file(target_filepath, url, target_dir=""):
     """
     download given file and eventually unzip it
     """
     logging_filename = target_filepath.split(os.sep)[-1]
     log.info('-' * 80)
     log.info('# Downloading %s file', logging_filename)
-    if is_zip:
+    if url.split('.')[-1] == 'zip':
         # build target-filepath based on last element of URL
         last_part = url.rsplit('/', 1)[-1]
         dl_file_path = os.path.join(USER_DL_DIR, last_part)
         # download URL to file
         download_url_to_file(url, dl_file_path)
-        # unpack it - should work on macOS and Windows
-        unzip(dl_file_path, USER_DL_DIR)
-        # delete .zip file
+
+        # if a target directory is given --> extract into that folder
+        if target_dir:
+            target_path = target_dir
+        else:
+            target_path = USER_DL_DIR
+
+        # unpack it
+        unzip(dl_file_path, target_path)
+
         os.remove(dl_file_path)
     else:
         # no zipping --> directly download to given target filepath
@@ -81,6 +91,33 @@ def get_osm_pbf_filepath_url(country):
     return map_file_path, url
 
 
+def download_tooling_win():
+    """
+    check for Windows tooling and download if not here already
+    this is done to bring down the filesize of the python module
+    """
+    # Windows
+    if platform.system() == "Windows":
+        if not os.path.isfile(OSMOSIS_WIN_FILE_PATH):
+            log.info('# Need to download Osmosis application for Windows')
+            download_file(OSMOSIS_WIN_FILE_PATH,
+                          'https://github.com/openstreetmap/osmosis/releases/download/0.48.3/osmosis-0.48.3.zip',
+                          get_tooling_win_path('Osmosis', in_user_dir=True))
+
+        mapwriter_plugin_path = os.path.join(USER_TOOLING_WIN_DIR,
+                                             'Osmosis', 'lib', 'default', 'mapsforge-map-writer-0.18.0-jar-with-dependencies.jar')
+        if not os.path.isfile(mapwriter_plugin_path):
+            log.info('# Need to download Osmosis mapwriter plugin for Windows')
+            download_file(mapwriter_plugin_path,
+                          'https://search.maven.org/remotecontent?filepath=org/mapsforge/mapsforge-map-writer/0.18.0/mapsforge-map-writer-0.18.0-jar-with-dependencies.jar')
+
+        if not os.path.isfile(get_tooling_win_path('osmfilter.exe', in_user_dir=True)):
+            log.info('# Need to download osmfilter application for Windows')
+
+            download_file(get_tooling_win_path('osmfilter.exe', in_user_dir=True),
+                          'http://m.m.i24.cc/osmfilter.exe')
+
+
 class Downloader:
     """
     This is the class to check and download maps / artifacts"
@@ -114,7 +151,7 @@ def download_geofabrik_file(self):
         download geofabrik file
         """
         download_file(GEOFABRIK_PATH,
-                      'https://download.geofabrik.de/index-v1.json', False)
+                      'https://download.geofabrik.de/index-v1.json')
 
         log.info('+ download geofabrik.json file: OK')
 
@@ -135,7 +172,7 @@ def download_files_if_needed(self):
         """
         if 'land_polygons' in self.need_to_dl:
             download_file(LAND_POLYGONS_PATH,
-                          'https://osmdata.openstreetmap.de/download/land-polygons-split-4326.zip', True)
+                          'https://osmdata.openstreetmap.de/download/land-polygons-split-4326.zip')
 
         # log.info('+ download land_polygons.shp file: OK')
 
@@ -223,7 +260,7 @@ def download_osm_pbf_file(self):
             try:
                 if item['download'] is True:
                     map_file_path, url = get_osm_pbf_filepath_url(country)
-                    download_file(map_file_path, url, False)
+                    download_file(map_file_path, url)
                     self.border_countries[country] = {
                         'map_file': map_file_path}
             except KeyError:

wahoomc/file_directory_functions.py

@@ -22,25 +22,8 @@ def unzip(source_filename, dest_dir):
     """
     unzip the given file into the given directory
     """
-    with zipfile.ZipFile(source_filename) as zip_file:
-        for member in zip_file.infolist():
-            # Path traversal defense copied from
-            # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
-            words = member.filename.split('/')
-            path = dest_dir
-            for word in words[:-1]:
-                while True:
-                    drive, word = os.path.splitdrive(word)
-                    head, word = os.path.split(  # pylint: disable=unused-variable
-                        word)
-                    if not drive:
-                        break
-                if word in (os.curdir, os.pardir, ''):
-                    continue
-                path = os.path.join(path, word)
-            if member.filename.split('/').pop():
-                member.filename = member.filename.split('/').pop()
-            zip_file.extract(member, path)
+    with zipfile.ZipFile(source_filename, 'r') as zip_ref:
+        zip_ref.extractall(dest_dir)
 
 
 def move_content(src_folder_name, dst_path):

wahoomc/main.py

@@ -11,6 +11,7 @@
 from wahoomc.setup_functions import initialize_work_directories, \
     check_installation_of_required_programs, write_config_file, \
     adjustments_due_to_breaking_changes, copy_jsons_from_repo_to_user
+from wahoomc.downloader import download_tooling_win
 
 from wahoomc.osm_maps_functions import OsmMaps
 from wahoomc.osm_maps_functions import OsmData
@@ -29,8 +30,11 @@ def run(run_level):
     logging.basicConfig(format='%(levelname)s:%(message)s',
                         level=logging.INFO)
 
+    # initializing work directories needs to be the first call,
+    # because other setup stuff relies on that (breaking changes)
     initialize_work_directories()
     adjustments_due_to_breaking_changes()
+    download_tooling_win()
     check_installation_of_required_programs()
 
     if run_level == 'init':

wahoomc/osm_maps_functions.py

@@ -8,7 +8,6 @@
 import glob
 import multiprocessing
 import os
-import struct
 import subprocess
 import sys
 import platform
@@ -26,6 +25,7 @@
 from wahoomc.constants import RESOURCES_DIR
 from wahoomc.constants import LAND_POLYGONS_PATH
 from wahoomc.constants import VERSION
+from wahoomc.constants import OSMOSIS_WIN_FILE_PATH
 
 from wahoomc.downloader import Downloader
 from wahoomc.geofabrik import Geofabrik
@@ -301,20 +301,12 @@ class OsmMaps:
     This is a OSM data class
     """
 
-    osmosis_win_file_path = get_tooling_win_path(
-        ['Osmosis', 'bin', 'osmosis.bat'])
-
     # Number of workers for the Osmosis read binary fast function
     workers = '1'
 
     def __init__(self, o_osm_data):
         self.o_osm_data = o_osm_data
-
-        if 8 * struct.calcsize("P") == 32:
-            self.osmconvert_path = get_tooling_win_path(['osmconvert'])
-        else:
-            self.osmconvert_path = get_tooling_win_path(
-                ['osmconvert64-0.8.8p'])
+        self.osmconvert_path = get_tooling_win_path('osmconvert')
 
         create_empty_directories(
             USER_OUTPUT_DIR, self.o_osm_data.tiles, self.o_osm_data.border_countries)
@@ -366,7 +358,7 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
                         or self.last_changed_is_identical_to_last_run(key) is False:
                     log.info(
                         '+ Filtering unwanted map objects out of map of %s', key)
-                    cmd = [get_tooling_win_path(['osmfilter'])]
+                    cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)]
                     cmd.append(out_file_o5m)
                     cmd.append(
                         '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"')
@@ -377,7 +369,7 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
                     run_subprocess_and_log_output(
                         cmd, f'! Error in OSMFilter with country: {key}')
 
-                    cmd = [get_tooling_win_path(['osmfilter'])]
+                    cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)]
                     cmd.append(out_file_o5m)
                     cmd.append(
                         '--keep="' + translate_tags_to_keep(
@@ -635,7 +627,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries):
 
             # Windows
             if platform.system() == "Windows":
-                cmd = [self.osmosis_win_file_path]
+                cmd = [OSMOSIS_WIN_FILE_PATH]
             # Non-Windows
             else:
                 cmd = ['osmosis']
@@ -692,7 +684,7 @@ def sort_osm_files(self, tile):
 
         for land in land_files:
             if platform.system() == "Windows":
-                cmd = [self.osmosis_win_file_path]
+                cmd = [OSMOSIS_WIN_FILE_PATH]
             else:
                 cmd = ['osmosis']
 
@@ -732,7 +724,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml):
 
             # Windows
             if platform.system() == "Windows":
-                cmd = [self.osmosis_win_file_path, '--rbf', merged_file,
+                cmd = [OSMOSIS_WIN_FILE_PATH, '--rbf', merged_file,
                        'workers=' + self.workers, '--mw', 'file='+out_file_map]
             # Non-Windows
             else:
@@ -757,7 +749,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml):
 
             # Windows
             if platform.system() == "Windows":
-                cmd = [get_tooling_win_path(['lzma']), 'e', out_file_map,
+                cmd = [get_tooling_win_path('lzma'), 'e', out_file_map,
                        out_file_map+'.lzma', f'-mt{threads}', '-d27', '-fb273', '-eos']
             # Non-Windows
             else:
@@ -819,7 +811,7 @@ def make_and_zip_files(self, extension, zip_folder):
         if zip_folder:
             # Windows
             if platform.system() == "Windows":
-                cmd = [get_tooling_win_path(['7za']), 'a', '-tzip']
+                cmd = [get_tooling_win_path('7za'), 'a', '-tzip']
 
                 cmd.extend(
                     [folder_name + '.zip', os.path.join(".", folder_name, "*")])