From 2c8696a58e550bbee4c5c758592f2ec27670202b Mon Sep 17 00:00:00 2001 From: Benjamin K <53038537+treee111@users.noreply.github.com> Date: Thu, 29 Sep 2022 23:17:32 +0200 Subject: [PATCH 01/13] [FEAT] Refactor a lot to decouple and make maintenance easier (focus osm_maps_functions.py) (#152) * import file_directory functions one by one - and not by importing the whole file "as" fd_fct * create OsmData object and insert into class OsmMaps * move check for -co and -xy to input.py * shuffle a lot * go throught calc_border_countries also when no border countries are wanted * break OsmData in small pieces * refactor border country calc + fix unittests * force_processing refactoring, all checks in OsmData now! - move to function process_input_of_the_tool - move corresponding stuff from check_and_download_files also - additional needed changes - mainiy in downloader.py - adjust tests & do nearly refactoring * import needed functions + pylint findings * refactor imports to "single import" and not the whole file * fix pylint finding * Bump to version v2.1.0a13 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8661e31d..b66013af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = wahoomc -version = 2.0.2 +version = 2.1.0a13 author = Benjamin Kreuscher author_email = benni.kreuscher@gmail.com description = Create maps for your Wahoo bike computer based on latest OSM maps From 972f81d59bf4f3ddb7a71b83dd2626db94268708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gro=C3=9F?= <100765265+zenziwerken@users.noreply.github.com> Date: Sun, 25 Sep 2022 21:07:06 +0200 Subject: [PATCH 02/13] Update osm_maps_functions.py According to the documentation https://wiki.openstreetmap.org/wiki/Osmfilter#Keep_only_specific_Tags tags have to be enclosed with >"< --- wahoomc/osm_maps_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 1b320849..9c9d61ae 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -344,7 +344,7 @@ def filter_tags_from_country_osm_pbf_files(self): cmd.append(out_file_o5m) cmd.append( '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('--keep-tags=all type= layer= "' + + cmd.append('--keep-tags="all type= layer= ' + translate_tags_to_keep(sys_platform=platform.system()) + '"') cmd.append('-o=' + out_file_o5m_filtered) @@ -356,7 +356,7 @@ def filter_tags_from_country_osm_pbf_files(self): cmd.append( '--keep="' + translate_tags_to_keep( name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('--keep-tags=all type= name= layer= "' + + cmd.append('--keep-tags="all type= name= layer= ' + translate_tags_to_keep( name_tags=True, sys_platform=platform.system()) + '"') cmd.append('-o=' + out_file_o5m_filtered_names) From 2392e970c3fd566b5726fbadb7a6bc556224e38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gro=C3=9F?= <100765265+zenziwerken@users.noreply.github.com> Date: Sun, 25 Sep 2022 21:12:25 +0200 Subject: [PATCH 03/13] Update osm_maps_functions.py Keep out_file_o5m if not force_processing to speed up build process when updating filtering tags / tag-mapping / tag-transform --- wahoomc/osm_maps_functions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 9c9d61ae..aa5134e8 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -326,7 +326,12 @@ def filter_tags_from_country_osm_pbf_files(self): out_file_o5m_filtered_names = os.path.join(USER_OUTPUT_DIR, f'outFileFiltered-{key}-Names.o5m') - if not os.path.isfile(out_file_o5m_filtered) or self.o_osm_data.force_processing is True: + if self.o_osm_data.force_processing is True: + os.remove(out_file_o5m) + else: + log.info('+ map of %s already in o5m format', key) + + if not os.path.isfile(out_file_o5m): log.info('+ Converting map of %s to o5m format', key) cmd = [self.osmconvert_path] cmd.extend(['-v', '--hash-memory=2500', '--complete-ways', @@ -338,6 +343,7 @@ def filter_tags_from_country_osm_pbf_files(self): run_subprocess_and_log_output( cmd, '! Error in OSMConvert with country: {key}') + if not os.path.isfile(out_file_o5m_filtered) or self.o_osm_data.force_processing is True: log.info( '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path(['osmfilter'])] @@ -364,8 +370,6 @@ def filter_tags_from_country_osm_pbf_files(self): run_subprocess_and_log_output( cmd, '! Error in OSMFilter with country: {key}') - os.remove(out_file_o5m) - val['filtered_file'] = out_file_o5m_filtered val['filtered_file_names'] = out_file_o5m_filtered_names From 668c0728fe852344ee3b7190742692b995b2f867 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Mon, 26 Sep 2022 21:08:40 +0200 Subject: [PATCH 04/13] finalization of @zenziwerken 's work pylint finding, comment --- wahoomc/osm_maps_functions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index aa5134e8..d2e5e882 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -308,7 +308,7 @@ def __init__(self, o_osm_data): create_empty_directories( USER_OUTPUT_DIR, self.o_osm_data.tiles) - def filter_tags_from_country_osm_pbf_files(self): + def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-statements """ Filter tags from country osm.pbf files """ @@ -329,8 +329,9 @@ def filter_tags_from_country_osm_pbf_files(self): if self.o_osm_data.force_processing is True: os.remove(out_file_o5m) else: - log.info('+ map of %s already in o5m format', key) + log.info('+ Map of %s already in o5m format', key) + # only create o5m file if not there already --> speeds up processing if one only wants to test tags / POIs if not os.path.isfile(out_file_o5m): log.info('+ Converting map of %s to o5m format', key) cmd = [self.osmconvert_path] From 2eaf88aa0d5be39ff4c37a42491c14067c5ccd3d Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Mon, 26 Sep 2022 21:37:07 +0200 Subject: [PATCH 05/13] catch error if file does not exist but should be deleted e.g. if processing -fp after switching from v2.0.2 to this version (Windows only) --- wahoomc/osm_maps_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index d2e5e882..43b4f5fe 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -327,7 +327,10 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st f'outFileFiltered-{key}-Names.o5m') if self.o_osm_data.force_processing is True: - os.remove(out_file_o5m) + try: + os.remove(out_file_o5m) + except OSError: + pass else: log.info('+ Map of %s already in o5m format', key) From 8a55e45d75ed39fccf0c31cd0549a0083311fc28 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Thu, 29 Sep 2022 21:08:19 +0200 Subject: [PATCH 06/13] adjust constants processing based on discussion in PR https://github.com/treee111/wahooMapsCreator/pull/150#issuecomment-1261475761 --- wahoomc/osm_maps_functions.py | 59 ++++++++++++++++------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 43b4f5fe..fb5301f7 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -326,16 +326,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st out_file_o5m_filtered_names = os.path.join(USER_OUTPUT_DIR, f'outFileFiltered-{key}-Names.o5m') - if self.o_osm_data.force_processing is True: - try: - os.remove(out_file_o5m) - except OSError: - pass - else: - log.info('+ Map of %s already in o5m format', key) - # only create o5m file if not there already --> speeds up processing if one only wants to test tags / POIs - if not os.path.isfile(out_file_o5m): + if not os.path.isfile(out_file_o5m) or self.o_osm_data.force_processing is True: log.info('+ Converting map of %s to o5m format', key) cmd = [self.osmconvert_path] cmd.extend(['-v', '--hash-memory=2500', '--complete-ways', @@ -346,33 +338,36 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st run_subprocess_and_log_output( cmd, '! Error in OSMConvert with country: {key}') + else: + log.info('+ Map of %s already in o5m format', key) - if not os.path.isfile(out_file_o5m_filtered) or self.o_osm_data.force_processing is True: - log.info( - '+ Filtering unwanted map objects out of map of %s', key) - cmd = [get_tooling_win_path(['osmfilter'])] - cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= layer= ' + - translate_tags_to_keep(sys_platform=platform.system()) + '"') - cmd.append('-o=' + out_file_o5m_filtered) + # filter out tags every time using the defined TAGS_TO_KEEP_UNIVERSAL constants + # because the result is different per constants + log.info( + '+ Filtering unwanted map objects out of map of %s', key) + cmd = [get_tooling_win_path(['osmfilter'])] + cmd.append(out_file_o5m) + cmd.append( + '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"') + cmd.append('--keep-tags="all type= layer= ' + + translate_tags_to_keep(sys_platform=platform.system()) + '"') + cmd.append('-o=' + out_file_o5m_filtered) - run_subprocess_and_log_output( - cmd, '! Error in OSMFilter with country: {key}') + run_subprocess_and_log_output( + cmd, '! Error in OSMFilter with country: {key}') - cmd = [get_tooling_win_path(['osmfilter'])] - cmd.append(out_file_o5m) - cmd.append( - '--keep="' + translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('--keep-tags="all type= name= layer= ' + - translate_tags_to_keep( - name_tags=True, sys_platform=platform.system()) + '"') - cmd.append('-o=' + out_file_o5m_filtered_names) + cmd = [get_tooling_win_path(['osmfilter'])] + cmd.append(out_file_o5m) + cmd.append( + '--keep="' + translate_tags_to_keep( + name_tags=True, sys_platform=platform.system()) + '"') + cmd.append('--keep-tags="all type= name= layer= ' + + translate_tags_to_keep( + name_tags=True, sys_platform=platform.system()) + '"') + cmd.append('-o=' + out_file_o5m_filtered_names) - run_subprocess_and_log_output( - cmd, '! Error in OSMFilter with country: {key}') + run_subprocess_and_log_output( + cmd, '! Error in OSMFilter with country: {key}') val['filtered_file'] = out_file_o5m_filtered val['filtered_file_names'] = out_file_o5m_filtered_names From 4c1b22c4a992a674ebf1bd4a01cba26d3f641020 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Thu, 29 Sep 2022 21:14:39 +0200 Subject: [PATCH 07/13] adjust tag-wahoo.xml processing based on discussion in PR https://github.com/treee111/wahooMapsCreator/pull/150#issuecomment-1261475761 --- wahoomc/osm_maps_functions.py | 75 ++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index fb5301f7..fad1db0c 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -716,50 +716,51 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml): '+ Creating map file for tile %s of %s for Coordinates: %s,%s', tile_count, len(self.o_osm_data.tiles), tile["x"], tile["y"]) out_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map') - if not os.path.isfile(out_file+'.lzma') or self.o_osm_data.force_processing is True: - merged_file = os.path.join(USER_OUTPUT_DIR, - f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf') - # Windows - if platform.system() == "Windows": - cmd = [self.osmosis_win_file_path, '--rbf', merged_file, - 'workers=' + self.workers, '--mw', 'file='+out_file] - # Non-Windows - else: - cmd = ['osmosis', '--rb', merged_file, - '--mw', 'file='+out_file] + # apply tag-wahoo xml every time because the result is different per user input + merged_file = os.path.join(USER_OUTPUT_DIR, + f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf') + # Windows + if platform.system() == "Windows": + cmd = [self.osmosis_win_file_path, '--rbf', merged_file, + 'workers=' + self.workers, '--mw', 'file='+out_file] + # Non-Windows + else: + cmd = ['osmosis', '--rb', merged_file, + '--mw', 'file='+out_file] + + cmd.append( + f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}') + cmd.append('zoom-interval-conf=10,0,17') + cmd.append(f'threads={threads}') + # add path to tag-wahoo xml file + try: cmd.append( - f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}') - cmd.append('zoom-interval-conf=10,0,17') - cmd.append(f'threads={threads}') - # add path to tag-wahoo xml file - try: - cmd.append( - f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}') - except TagWahooXmlNotFoundError: - log.error( - 'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml) - sys.exit() + f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}') + except TagWahooXmlNotFoundError: + log.error( + 'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml) + sys.exit() - run_subprocess_and_log_output( - cmd, f'Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?') + run_subprocess_and_log_output( + cmd, f'Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?') - # Windows - if platform.system() == "Windows": - cmd = [get_tooling_win_path(['lzma']), 'e', out_file, - out_file+'.lzma', f'-mt{threads}', '-d27', '-fb273', '-eos'] - # Non-Windows - else: - # force overwrite of output file and (de)compress links - cmd = ['lzma', out_file, '-f'] + # Windows + if platform.system() == "Windows": + cmd = [get_tooling_win_path(['lzma']), 'e', out_file, + out_file+'.lzma', f'-mt{threads}', '-d27', '-fb273', '-eos'] + # Non-Windows + else: + # force overwrite of output file and (de)compress links + cmd = ['lzma', out_file, '-f'] - # --keep: do not delete source file - if save_cruiser: - cmd.append('--keep') + # --keep: do not delete source file + if save_cruiser: + cmd.append('--keep') - run_subprocess_and_log_output( - cmd, f'! Error creating map files for tile: {tile["x"]},{tile["y"]}') + run_subprocess_and_log_output( + cmd, f'! Error creating map files for tile: {tile["x"]},{tile["y"]}') # Create "tile present" file with open(out_file + '.lzma.17', mode='wb') as tile_present_file: From 9cb1151b418616fcceead001a8ad9a622d495cee Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Thu, 29 Sep 2022 22:08:28 +0200 Subject: [PATCH 08/13] adjust splitting to tiles processing based on discussion in PR https://github.com/treee111/wahooMapsCreator/pull/150#issuecomment-1261475761 --- wahoomc/osm_maps_functions.py | 89 ++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index fad1db0c..d7048b16 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -528,58 +528,59 @@ def split_filtered_country_files_to_tiles(self): f'{tile["x"]}', f'{tile["y"]}', f'split-{country}-names.osm.pbf') out_merged = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf') - if not os.path.isfile(out_merged) or self.o_osm_data.force_processing is True: - # Windows - if platform.system() == "Windows": - cmd = [self.osmconvert_path, - '-v', '--hash-memory=2500'] - cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' + - ',' + f'{tile["right"]}' + ',' + f'{tile["top"]}') - cmd.extend( - ['--complete-ways', '--complete-multipolygons', '--complete-boundaries']) - cmd.append(val['filtered_file']) - cmd.append('-o='+out_file) - run_subprocess_and_log_output( - cmd, f'! Error in Osmosis with country: {country}. Win/out_file') + # split filtered country files to tiles every time because the result is different per constants + # Windows + if platform.system() == "Windows": + cmd = [self.osmconvert_path, + '-v', '--hash-memory=2500'] + cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' + + ',' + f'{tile["right"]}' + ',' + f'{tile["top"]}') + cmd.extend( + ['--complete-ways', '--complete-multipolygons', '--complete-boundaries']) + cmd.append(val['filtered_file']) + cmd.append('-o='+out_file) - cmd = [self.osmconvert_path, - '-v', '--hash-memory=2500'] - cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' + - ',' + f'{tile["right"]}' + ',' + f'{tile["top"]}') - cmd.extend( - ['--complete-ways', '--complete-multipolygons', '--complete-boundaries']) - cmd.append(val['filtered_file_names']) - cmd.append('-o='+out_file_names) + run_subprocess_and_log_output( + cmd, f'! Error in Osmosis with country: {country}. Win/out_file') - run_subprocess_and_log_output( - cmd, '! Error in Osmosis with country: {country}. Win/out_file_names') + cmd = [self.osmconvert_path, + '-v', '--hash-memory=2500'] + cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' + + ',' + f'{tile["right"]}' + ',' + f'{tile["top"]}') + cmd.extend( + ['--complete-ways', '--complete-multipolygons', '--complete-boundaries']) + cmd.append(val['filtered_file_names']) + cmd.append('-o='+out_file_names) - # Non-Windows - else: - cmd = ['osmium', 'extract'] - cmd.extend( - ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}']) - cmd.append(val['filtered_file']) - cmd.extend(['-s', 'smart']) - cmd.extend(['-o', out_file]) - cmd.extend(['--overwrite']) + run_subprocess_and_log_output( + cmd, '! Error in Osmosis with country: {country}. Win/out_file_names') - run_subprocess_and_log_output( - cmd, '! Error in Osmosis with country: {country}. macOS/out_file') + # Non-Windows + else: + cmd = ['osmium', 'extract'] + cmd.extend( + ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}']) + cmd.append(val['filtered_file']) + cmd.extend(['-s', 'smart']) + cmd.extend(['-o', out_file]) + cmd.extend(['--overwrite']) - cmd = ['osmium', 'extract'] - cmd.extend( - ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}']) - cmd.append(val['filtered_file_names']) - cmd.extend(['-s', 'smart']) - cmd.extend(['-o', out_file_names]) - cmd.extend(['--overwrite']) + run_subprocess_and_log_output( + cmd, '! Error in Osmosis with country: {country}. macOS/out_file') + + cmd = ['osmium', 'extract'] + cmd.extend( + ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}']) + cmd.append(val['filtered_file_names']) + cmd.extend(['-s', 'smart']) + cmd.extend(['-o', out_file_names]) + cmd.extend(['--overwrite']) - run_subprocess_and_log_output( - cmd, '! Error in Osmosis with country: {country}. macOS/out_file_names') + run_subprocess_and_log_output( + cmd, '! Error in Osmosis with country: {country}. macOS/out_file_names') - log.info(val['filtered_file']) + log.info(val['filtered_file']) tile_count += 1 From 60444be0e4e5885066176e58caae021593be27cc Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Thu, 29 Sep 2022 22:30:41 +0200 Subject: [PATCH 09/13] adjust merging with land and sea processing based on discussion in PR https://github.com/treee111/wahooMapsCreator/pull/150#issuecomment-1261475761 --- wahoomc/osm_maps_functions.py | 94 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index d7048b16..970df711 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -604,60 +604,60 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries): land_files = glob.glob(os.path.join(out_tile_dir, 'land*.osm')) - if not os.path.isfile(out_file) or self.o_osm_data.force_processing is True: - # sort land* osm files - self.sort_osm_files(tile) + # merge splitted tiles with land and sea every time because the result is different per constants + # sort land* osm files + self.sort_osm_files(tile) - # Windows - if platform.system() == "Windows": - cmd = [self.osmosis_win_file_path] - loop = 0 - # loop through all countries of tile, if border-countries should be processed. - # if border-countries should not be processed, only process the "entered" country - for country in tile['countries']: - if process_border_countries or country in self.o_osm_data.border_countries: - cmd.append('--rbf') - cmd.append(os.path.join( - out_tile_dir, f'split-{country}.osm.pbf')) - cmd.append('workers=' + self.workers) - if loop > 0: - cmd.append('--merge') - - cmd.append('--rbf') - cmd.append(os.path.join( - out_tile_dir, f'split-{country}-names.osm.pbf')) - cmd.append('workers=' + self.workers) + # Windows + if platform.system() == "Windows": + cmd = [self.osmosis_win_file_path] + loop = 0 + # loop through all countries of tile, if border-countries should be processed. + # if border-countries should not be processed, only process the "entered" country + for country in tile['countries']: + if process_border_countries or country in self.o_osm_data.border_countries: + cmd.append('--rbf') + cmd.append(os.path.join( + out_tile_dir, f'split-{country}.osm.pbf')) + cmd.append('workers=' + self.workers) + if loop > 0: cmd.append('--merge') - loop += 1 + cmd.append('--rbf') + cmd.append(os.path.join( + out_tile_dir, f'split-{country}-names.osm.pbf')) + cmd.append('workers=' + self.workers) + cmd.append('--merge') - for land in land_files: - cmd.extend( - ['--rx', 'file='+os.path.join(out_tile_dir, f'{land}'), '--s', '--m']) + loop += 1 + + for land in land_files: cmd.extend( - ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) - cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, - 'tunnel-transform.xml'), '--wb', out_file, 'omitmetadata=true']) + ['--rx', 'file='+os.path.join(out_tile_dir, f'{land}'), '--s', '--m']) + cmd.extend( + ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) + cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, + 'tunnel-transform.xml'), '--wb', out_file, 'omitmetadata=true']) - # Non-Windows - else: - cmd = ['osmium', 'merge', '--overwrite'] - # loop through all countries of tile, if border-countries should be processed. - # if border-countries should not be processed, only process the "entered" country - for country in tile['countries']: - if process_border_countries or country in self.o_osm_data.border_countries: - cmd.append(os.path.join( - out_tile_dir, f'split-{country}.osm.pbf')) - cmd.append(os.path.join( - out_tile_dir, f'split-{country}-names.osm.pbf')) - - for land in land_files: - cmd.append(land) - cmd.append(os.path.join(out_tile_dir, 'sea.osm')) - cmd.extend(['-o', out_file]) + # Non-Windows + else: + cmd = ['osmium', 'merge', '--overwrite'] + # loop through all countries of tile, if border-countries should be processed. + # if border-countries should not be processed, only process the "entered" country + for country in tile['countries']: + if process_border_countries or country in self.o_osm_data.border_countries: + cmd.append(os.path.join( + out_tile_dir, f'split-{country}.osm.pbf')) + cmd.append(os.path.join( + out_tile_dir, f'split-{country}-names.osm.pbf')) + + for land in land_files: + cmd.append(land) + cmd.append(os.path.join(out_tile_dir, 'sea.osm')) + cmd.extend(['-o', out_file]) - run_subprocess_and_log_output( - cmd, f'! Error in Osmosis with tile: {tile["x"]},{tile["y"]}') + run_subprocess_and_log_output( + cmd, f'! Error in Osmosis with tile: {tile["x"]},{tile["y"]}') tile_count += 1 From de0e856b3ac94c530e5e97fa383ead978e2b699e Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Thu, 29 Sep 2022 23:33:28 +0200 Subject: [PATCH 10/13] fix pylint finding --- wahoomc/osm_maps_functions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 970df711..5d05c4ec 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -526,8 +526,6 @@ def split_filtered_country_files_to_tiles(self): f'{tile["x"]}', f'{tile["y"]}', f'split-{country}.osm.pbf') out_file_names = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', f'split-{country}-names.osm.pbf') - out_merged = os.path.join(USER_OUTPUT_DIR, - f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf') # split filtered country files to tiles every time because the result is different per constants # Windows From 4120beeeb18e68e372f925d84e0d00d743cf9097 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sun, 2 Oct 2022 20:49:48 +0200 Subject: [PATCH 11/13] harmonize comments --- wahoomc/osm_maps_functions.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 5d05c4ec..ff77821e 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -326,7 +326,8 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st out_file_o5m_filtered_names = os.path.join(USER_OUTPUT_DIR, f'outFileFiltered-{key}-Names.o5m') - # only create o5m file if not there already --> speeds up processing if one only wants to test tags / POIs + # only create o5m file if not there already or force processing (no user input possible) + # --> speeds up processing if one only wants to test tags / POIs if not os.path.isfile(out_file_o5m) or self.o_osm_data.force_processing is True: log.info('+ Converting map of %s to o5m format', key) cmd = [self.osmconvert_path] @@ -342,7 +343,7 @@ def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-st log.info('+ Map of %s already in o5m format', key) # filter out tags every time using the defined TAGS_TO_KEEP_UNIVERSAL constants - # because the result is different per constants + # because the result is different per constants (user input) log.info( '+ Filtering unwanted map objects out of map of %s', key) cmd = [get_tooling_win_path(['osmfilter'])] @@ -527,7 +528,7 @@ def split_filtered_country_files_to_tiles(self): out_file_names = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', f'split-{country}-names.osm.pbf') - # split filtered country files to tiles every time because the result is different per constants + # split filtered country files to tiles every time because the result is different per constants (user input) # Windows if platform.system() == "Windows": cmd = [self.osmconvert_path, @@ -602,7 +603,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries): land_files = glob.glob(os.path.join(out_tile_dir, 'land*.osm')) - # merge splitted tiles with land and sea every time because the result is different per constants + # merge splitted tiles with land and sea every time because the result is different per constants (user input) # sort land* osm files self.sort_osm_files(tile) @@ -716,7 +717,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml): out_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map') - # apply tag-wahoo xml every time because the result is different per user input + # apply tag-wahoo xml every time because the result is different per .xml file (user input) merged_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf') From 8d93b5185609ef9e2b5e6c43651154ac8893815b Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sun, 2 Oct 2022 21:46:18 +0200 Subject: [PATCH 12/13] Bump to version v2.1.0a14 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b66013af..ac4c12f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = wahoomc -version = 2.1.0a13 +version = 2.1.0a14 author = Benjamin Kreuscher author_email = benni.kreuscher@gmail.com description = Create maps for your Wahoo bike computer based on latest OSM maps From 7068867da9bbdd38cd2b34b2cd0900759af52509 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Mon, 3 Oct 2022 20:32:53 +0200 Subject: [PATCH 13/13] Revert to version v2.0.2 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ac4c12f0..8661e31d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = wahoomc -version = 2.1.0a14 +version = 2.0.2 author = Benjamin Kreuscher author_email = benni.kreuscher@gmail.com description = Create maps for your Wahoo bike computer based on latest OSM maps