From 03f31a263b0441849b3cbe626e5589346be049e0 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Tue, 14 Feb 2023 08:36:32 +0100 Subject: [PATCH 01/21] variables are empty at that time should make it clearer like this --- wahoomc/osm_maps_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 3a514ed1..329dc822 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -148,7 +148,7 @@ def process_input_of_the_tool(self, o_input_data): """ o_downloader = Downloader( - o_input_data.max_days_old, o_input_data.force_download, self.tiles, self.border_countries) + o_input_data.max_days_old, o_input_data.force_download, None, None) # takeover what is given by user first for force_processing self.force_processing = o_input_data.force_processing From a4774397235cfbe9f43a31d21c1e693c317e87c9 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Tue, 14 Feb 2023 08:41:12 +0100 Subject: [PATCH 02/21] unittests for existing functionality --- tests/test_constants_geofabrik.py | 94 +++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/test_constants_geofabrik.py diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py new file mode 100644 index 00000000..fa9e0d5e --- /dev/null +++ b/tests/test_constants_geofabrik.py @@ -0,0 +1,94 @@ +""" +tests for the constants geofabrik & geofabrik file +""" +import os +import unittest + +# import custom python packages +from wahoomc import file_directory_functions as fd_fct +from wahoomc.downloader import Downloader +from wahoomc import constants +from wahoomc.geofabrik import find_geofbrik_parent +from wahoomc.constants_functions import translate_country_input_to_geofabrik + +skip_countries_for_geofabrik = [ + 'clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', + 'curacao', 'bonaire_saint_eustatius_and_saba', 'falkland_islands', 'french_guiana', + 'aruba', 'united_states_minor_outlying_islands', 'french_polynesia', + 'norfolk_island', 'wallis_and_futuna', + 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', + 'guam', 'montserrat', 'bermuda', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', + 'cayman_islands', 'haiti', 'saint_lucia', 'saint_kitts_and_nevis', 'dominican_republic', + 'turks_and_caicos_islands', 'antigua_and_barbuda', 'comoros', 'french_southern_territories'] + + +class TestConstantsGeofabrik(unittest.TestCase): + """ + tests for constants in geofabrik context + """ + + def setUp(self): + if not os.path.isfile(constants.GEOFABRIK_PATH): + o_downloader = Downloader(24, False) + o_downloader.download_geofabrik_file() + + with open(constants.GEOFABRIK_PATH, encoding='utf8') as file_handle: + self.geofabrik_json_data = geojson.load(file_handle) + file_handle.close() + + def test_if_json_countries_exist_in_geofabrik(self): + """ + go through all files in the wahoo_mc/resources/json directory + - check if each country does also exist in geofabrik + - some countries are skipped because they do not exist in geofabrik + """ + for folder in fd_fct.get_folders_in_folder(os.path.join(constants.RESOURCES_DIR, 'json')): + for file in fd_fct.get_files_in_folder(os.path.join(constants.RESOURCES_DIR, 'json', folder)): + country = os.path.splitext(file)[0] + if country in skip_countries_for_geofabrik: + continue + + parent, child = find_geofbrik_parent(country) + if not child: + parent, child = find_geofbrik_parent( + translate_country_input_to_geofabrik(country)) + + if not child: + country = country.replace('_', '-') + parent, child = find_geofbrik_parent(country) + + if not child: + parent, child = find_geofbrik_parent( + 'us/'+country) + + if not child: + self.assertEqual(country, child) + + else: + self.assertEqual(country, child) + + def test_regions_of_geofabrik(self): + """ + go through all files in the wahoo_mc/resources/json directory + - check if each country does also exist in geofabrik + - some countries are skipped because they do not exist in geofabrik + """ + + id_with_no_parent_geofabrik = [] + + for feature in self.geofabrik_json_data.features: + try: + feature.properties['parent'] + except KeyError: + id_with_no_parent_geofabrik.append(feature.properties['id']) + + geofabrik_regions_w_russia = constants.geofabrik_regions + geofabrik_regions_w_russia.append( + 'russia') + + self.assertCountEqual(geofabrik_regions_w_russia, + id_with_no_parent_geofabrik) + + +if __name__ == '__main__': + unittest.main() From f601212c9c04400a6a86457b3f13161d3dc16bcd Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Tue, 14 Feb 2023 14:57:13 +0100 Subject: [PATCH 03/21] geofabrik .json file as own class, move gefabrik functions - less cluttered (access) - only one read location of the .json file - download might be placed to another place or changed --- tests/test_constants_geofabrik.py | 18 +++++++------- wahoomc/constants_functions.py | 40 ++++++++++++++++++++++++++++++ wahoomc/downloader.py | 7 ++++++ wahoomc/geofabrik.py | 41 +++++-------------------------- 4 files changed, 62 insertions(+), 44 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index fa9e0d5e..dd26f1d0 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -8,7 +8,7 @@ from wahoomc import file_directory_functions as fd_fct from wahoomc.downloader import Downloader from wahoomc import constants -from wahoomc.geofabrik import find_geofbrik_parent +from wahoomc.constants_functions import GeofabrikJson from wahoomc.constants_functions import translate_country_input_to_geofabrik skip_countries_for_geofabrik = [ @@ -32,9 +32,7 @@ def setUp(self): o_downloader = Downloader(24, False) o_downloader.download_geofabrik_file() - with open(constants.GEOFABRIK_PATH, encoding='utf8') as file_handle: - self.geofabrik_json_data = geojson.load(file_handle) - file_handle.close() + self.o_geofabrik_json = GeofabrikJson() def test_if_json_countries_exist_in_geofabrik(self): """ @@ -48,17 +46,19 @@ def test_if_json_countries_exist_in_geofabrik(self): if country in skip_countries_for_geofabrik: continue - parent, child = find_geofbrik_parent(country) + parent, child = self.o_geofabrik_json.find_geofbrik_parent( + country) if not child: - parent, child = find_geofbrik_parent( + parent, child = self.o_geofabrik_json.find_geofbrik_parent( translate_country_input_to_geofabrik(country)) if not child: country = country.replace('_', '-') - parent, child = find_geofbrik_parent(country) + parent, child = self.o_geofabrik_json.find_geofbrik_parent( + country) if not child: - parent, child = find_geofbrik_parent( + parent, child = self.o_geofabrik_json.find_geofbrik_parent( 'us/'+country) if not child: @@ -76,7 +76,7 @@ def test_regions_of_geofabrik(self): id_with_no_parent_geofabrik = [] - for feature in self.geofabrik_json_data.features: + for feature in self.o_geofabrik_json.json_data.features: try: feature.properties['parent'] except KeyError: diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 781dbb86..135e3afc 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -8,6 +8,7 @@ import logging import os import struct +import geojson # import custom python packages from wahoomc import constants @@ -15,6 +16,7 @@ from wahoomc.constants import TOOLING_WIN_DIR from wahoomc.constants import USER_CONFIG_DIR from wahoomc.constants import USER_TOOLING_WIN_DIR +from wahoomc.constants import GEOFABRIK_PATH from wahoomc.file_directory_functions import read_json_file_generic log = logging.getLogger('main-logger') @@ -28,6 +30,44 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" +class GeofabrikJson: + """ + This is a Geofabrik .json processing class for constants in the Geofabrik .json file + """ + + def __init__(self): + + with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: + self.json_data = geojson.load(file_handle) + file_handle.close() + + def find_geofbrik_parent(self, name): + """ + Get the parent map/region of a region from the already loaded json data + """ + for feature in self.json_data.features: + props = feature.properties + ident_no = props.get('id', '') + if ident_no != name: + continue + return (props.get('parent', ''), props.get('id', '')) + return None, None + + def find_geofbrik_url(self, name): + """ + Get the map download url from a region with the already loaded json data + """ + for feature in self.json_data.features: + props = feature.properties + ident_no = props.get('id', '') + if ident_no != name: + continue + #print (props.get('urls', '')) + wurls = props.get('urls', '') + return wurls.get('pbf', '') + return None + + def get_region_of_country(county): """ returns the region / continent of a given country diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index dda81a23..fca63ac2 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -136,6 +136,13 @@ def download_tooling(): os.makedirs(os.path.dirname(mapwriter_plugin_path), exist_ok=True) download_file(mapwriter_plugin_path, mapwriter_plugin_url) + # download geofabrik json as this will be needed always + # because of the .json file replacement by geofabrik + o_downloader = Downloader(24, False) + + if o_downloader.should_geofabrik_file_be_downloaded(): + o_downloader.download_geofabrik_file() + def get_latest_pypi_version(): """ diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index 94c98d8f..6bd6a426 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -14,6 +14,7 @@ # import custom python packages from wahoomc.constants import GEOFABRIK_PATH from wahoomc.constants import special_regions, geofabrik_regions, block_download +from wahoomc.constants_functions import GeofabrikJson log = logging.getLogger('main-logger') @@ -159,34 +160,6 @@ def geom(wanted): return None, None -def find_geofbrik_parent(name, geofabrik_json): - """ - Get the parent map/region of a region from the already loaded json data - """ - for feature in geofabrik_json.features: - props = feature.properties - ident_no = props.get('id', '') - if ident_no != name: - continue - return (props.get('parent', ''), props.get('id', '')) - return None, None - - -def find_geofbrik_url(name, geofabrik_json): - """ - Get the map download url from a region with the already loaded json data - """ - for feature in geofabrik_json.features: - props = feature.properties - ident_no = props.get('id', '') - if ident_no != name: - continue - #print (props.get('urls', '')) - wurls = props.get('urls', '') - return wurls.get('pbf', '') - return None - - def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): """ Find the maps to download from Geofabrik for a given range of tiles @@ -197,9 +170,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): """ output = [] - with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: - geofabrik_json_data = geojson.load(file_handle) - file_handle.close() + o_geofabrik_json = GeofabrikJson() # itterate through tiles and find Geofabrik regions that are in the tiles counter = 1 @@ -226,7 +197,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): must_download_urls = [] # itterate through countries/regions in the geofabrik json file - for regions in geofabrik_json_data.features: + for regions in o_geofabrik_json.json_data.features: props = regions.properties parent = props.get('parent', '') regionname = props.get('id', '') @@ -255,8 +226,8 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): x_value = 0 # handle sub-sub-regions like unterfranken->bayern->germany while parent not in geofabrik_regions: - parent, child = find_geofbrik_parent( - parent, geofabrik_json_data) + parent, child = o_geofabrik_json.find_geofbrik_parent( + parent) if parent in geofabrik_regions: parent = child break @@ -268,7 +239,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): if parent not in must_download_maps: must_download_maps.append(parent) must_download_urls.append( - find_geofbrik_url(parent, geofabrik_json_data)) + o_geofabrik_json.find_geofbrik_url(parent)) #parent_added = 1 else: if regionname not in must_download_maps: From 22979a60bcd3107633723793ff12eb63b3f8b6ab Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Wed, 15 Feb 2023 22:04:01 +0100 Subject: [PATCH 04/21] correct json file to geofabrik country comparison - dont use function translate_country_input_to_geofabrik as it make things harder to understand - calc relevant countries in setUp. these should be valid for the whole class - remove all countries which have no id in geofabrik file, makes most sense here --- tests/test_constants_geofabrik.py | 80 +++++++++++++++++-------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index dd26f1d0..d164cb1f 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -9,17 +9,13 @@ from wahoomc.downloader import Downloader from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson -from wahoomc.constants_functions import translate_country_input_to_geofabrik -skip_countries_for_geofabrik = [ - 'clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', - 'curacao', 'bonaire_saint_eustatius_and_saba', 'falkland_islands', 'french_guiana', - 'aruba', 'united_states_minor_outlying_islands', 'french_polynesia', - 'norfolk_island', 'wallis_and_futuna', - 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', - 'guam', 'montserrat', 'bermuda', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', - 'cayman_islands', 'haiti', 'saint_lucia', 'saint_kitts_and_nevis', 'dominican_republic', - 'turks_and_caicos_islands', 'antigua_and_barbuda', 'comoros', 'french_southern_territories'] +# json countries with no geofabrik id partner +json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', + 'curacao', 'bonaire_saint_eustatius_and_saba', 'falkland_islands', 'french_guiana', + 'aruba', 'united_states_minor_outlying_islands', 'french_polynesia', + 'norfolk_island', 'wallis_and_futuna', 'northern_mariana_islands', 'paracel_islands', 'united_arab_emirates', 'kuwait', 'qatar', 'spratly_islands', 'singapore', 'brunei', 'bahrain', 'macao', 'cocos_islands', 'christmas_island', 'palestina', 'malaysia', 'saudi_arabia', 'british_indian_ocean_territory', 'israel', 'oman', 'hong_kong', 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', 'guam', 'commonwealth_of_the_northern_mariana_islands', + 'american_samoa', 'united_states_virgin_islands', 'svalbard_and_jan_mayen', 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'vatican_city', 'ireland', 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'montserrat', 'bermuda', 'virgin_islands_u.s.', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', 'anguilla', 'saint-martin', 'cayman_islands', 'sint_maarten', 'haiti', 'saint_lucia', 'british_virgin_islands', 'saint_kitts_and_nevis', 'dominican_republic', 'turks_and_caicos_islands', 'antigua_and_barbuda', 'gambia', 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'comoros', 'republic_of_congo', 'democratic_republic_of_the_congo', 'senegal', 'french_southern_territories'] class TestConstantsGeofabrik(unittest.TestCase): @@ -34,38 +30,26 @@ def setUp(self): self.o_geofabrik_json = GeofabrikJson() + self.relevant_countries = [] + + # calc relevant countries in constructor. these should be valid for the whole class + for folder in fd_fct.get_folders_in_folder(os.path.join(constants.RESOURCES_DIR, 'json')): + for file in fd_fct.get_files_in_folder(os.path.join(constants.RESOURCES_DIR, 'json', folder)): + country = os.path.splitext(file)[0] + if country not in json_file_countries_without_geofabrik_id: + self.relevant_countries.append(country) + def test_if_json_countries_exist_in_geofabrik(self): """ go through all files in the wahoo_mc/resources/json directory - check if each country does also exist in geofabrik - some countries are skipped because they do not exist in geofabrik """ - for folder in fd_fct.get_folders_in_folder(os.path.join(constants.RESOURCES_DIR, 'json')): - for file in fd_fct.get_files_in_folder(os.path.join(constants.RESOURCES_DIR, 'json', folder)): - country = os.path.splitext(file)[0] - if country in skip_countries_for_geofabrik: - continue - - parent, child = self.o_geofabrik_json.find_geofbrik_parent( - country) - if not child: - parent, child = self.o_geofabrik_json.find_geofbrik_parent( - translate_country_input_to_geofabrik(country)) - - if not child: - country = country.replace('_', '-') - parent, child = self.o_geofabrik_json.find_geofbrik_parent( - country) - - if not child: - parent, child = self.o_geofabrik_json.find_geofbrik_parent( - 'us/'+country) + for country in self.relevant_countries: + child = self.get_geofabrik_id_by_json_file_country(country) - if not child: - self.assertEqual(country, child) - - else: - self.assertEqual(country, child) + self.assertIn(child, {country.replace( + '_', '-'), 'us/'+country.replace('_', '-')}) def test_regions_of_geofabrik(self): """ @@ -89,6 +73,32 @@ def test_regions_of_geofabrik(self): self.assertCountEqual(geofabrik_regions_w_russia, id_with_no_parent_geofabrik) + def get_geofabrik_id_by_json_file_country(self, country): + """ + get geofabrik id by country .json filename + """ + # get geofabrik id by country .json filename + # 1. raw + # parent, child = self.o_geofabrik_json.find_geofbrik_parent( + # country) + # if not child: + + # get geofabrik id by country .json filename + # 1. '_' replaced by '-' + child = self.o_geofabrik_json.find_geofbrik_parent( + country.replace('_', '-'))[1] + if child: + return child + + # 2. 'us/' prefix and '_' replaced by '-' + child = self.o_geofabrik_json.find_geofbrik_parent( + 'us/'+country.replace('_', '-'))[1] + + if child: + return child + + return None + if __name__ == '__main__': unittest.main() From fab8f8aed433d5a400e5b1159e58e03855ce2336 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Wed, 15 Feb 2023 22:09:37 +0100 Subject: [PATCH 05/21] unittest: compare built URL with geofabrik URL --- tests/test_constants_geofabrik.py | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index d164cb1f..8a28c54e 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -7,6 +7,7 @@ # import custom python packages from wahoomc import file_directory_functions as fd_fct from wahoomc.downloader import Downloader +from wahoomc.downloader import get_osm_pbf_filepath_url from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson @@ -73,6 +74,74 @@ def test_regions_of_geofabrik(self): self.assertCountEqual(geofabrik_regions_w_russia, id_with_no_parent_geofabrik) + def test_gofabrik_url_against_built_url(self): + """ + go through all files in the wahoo_mc/resources/json directory + - compare the built URL via existing coding with the url out of the geofabrik json file + - some countries are skipped because they do not exist in geofabrik + - in addition + - skip countries if there is a geofabrik URL (because this URL is different to the built one) + - skip countries which do not have a direct geofabrik json entry, mostly they are in another country/region + """ + for country in self.relevant_countries: + built_url = None + geofabrik_url = None + + # build URL with existing coding + try: + map_file_path, built_url = get_osm_pbf_filepath_url(country) + except SystemExit: + pass + + # grab URL from geofabrik json + geofabrik_url = self.o_geofabrik_json.find_geofbrik_url(country) + + if not geofabrik_url: + # check with replaced _ by - + geofabrik_url = self.o_geofabrik_json.find_geofbrik_url( + country.replace('_', '-')) + if not geofabrik_url: + # check with replaced _ by - AND prefix us/ + geofabrik_url = self.o_geofabrik_json.find_geofbrik_url( + 'us/'+country.replace('_', '-')) + + skip_countries_built_url_is_false = [ + 'papua_new_guinea', 'east_timor', + 'antarctica', # built url is false + 'georgia' # will issue north-america/us but europe is right. Reason: order in get_geofabrik_region_of_country + ] + + # skip_countries_for_url_comparison = [ + # 'united_arab_emirates', 'kuwait', 'qatar', 'bahrain', 'saudi_arabia', 'oman', # gcc-states + # 'malaysia', 'singapore', 'brunei', # malaysia-singapore-brunei + # 'christmas_island', # indonesia + # 'paracel_islands', 'spratly_islands', 'british_indian_ocean_territory', # asia + # 'macao', 'hong_kong', # china + # 'palestina', 'israel', # israel-and-palestine + # 'northern_mariana_islands', 'cocos_islands', 'commonwealth_of_the_northern_mariana_islands', 'american_samoa', # australia + # 'antarctica', # built url is false + # 'georgia', # will issue north-america/us but europe is right because of the order in get_geofabrik_region_of_country + # 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'san_marino', 'vatican_city', 'ireland', # europe + # 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'svalbard_and_jan_mayen', # europe + # 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'democratic_republic_of_the_congo'] # africa + + if built_url != geofabrik_url: + # # either it is a country to skip + # if (country in skip_countries_for_url_comparison_if_geofabrik_url and geofabrik_url) \ + # or country in skip_countries_for_url_comparison: + # continue + + # or there is no built url but a geofabrik one - ok! + # for some countries the built url is false + if built_url is None or \ + country in skip_countries_built_url_is_false: + self.assertTrue(geofabrik_url, msg='country: '+country) + continue + + # "normal" compare, also if both are empty/ None + self.assertEqual(built_url, geofabrik_url, + msg='country: '+country) + def get_geofabrik_id_by_json_file_country(self, country): """ get geofabrik id by country .json filename From 1bb71294344cacee7dc2e9efc3fd4ba85f886ac7 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Wed, 15 Feb 2023 23:03:43 +0100 Subject: [PATCH 06/21] refactor geofabrik class - for better access don't read the whole geofabrik and loop through it - refactor get functions to reflect that and easier reading --- tests/test_constants_geofabrik.py | 49 +++++++++++++++++++++---- wahoomc/constants_functions.py | 59 +++++++++++++++++++++---------- wahoomc/geofabrik.py | 6 ++-- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 8a28c54e..aec66f25 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -60,8 +60,10 @@ def test_regions_of_geofabrik(self): """ id_with_no_parent_geofabrik = [] + regions_geofabrik = [] - for feature in self.o_geofabrik_json.json_data.features: + # check against (new) raw representation of geofabrik json + for feature in self.o_geofabrik_json.raw_json.features: try: feature.properties['parent'] except KeyError: @@ -74,6 +76,12 @@ def test_regions_of_geofabrik(self): self.assertCountEqual(geofabrik_regions_w_russia, id_with_no_parent_geofabrik) + # also check against (new) created dict + for region in self.o_geofabrik_json.geofabrik_region_overview: + regions_geofabrik.append(region) + self.assertCountEqual(geofabrik_regions_w_russia, + regions_geofabrik) + def test_gofabrik_url_against_built_url(self): """ go through all files in the wahoo_mc/resources/json directory @@ -94,15 +102,15 @@ def test_gofabrik_url_against_built_url(self): pass # grab URL from geofabrik json - geofabrik_url = self.o_geofabrik_json.find_geofbrik_url(country) + geofabrik_url = self.o_geofabrik_json.get_geofabrik_url(country) if not geofabrik_url: # check with replaced _ by - - geofabrik_url = self.o_geofabrik_json.find_geofbrik_url( + geofabrik_url = self.o_geofabrik_json.get_geofabrik_url( country.replace('_', '-')) if not geofabrik_url: # check with replaced _ by - AND prefix us/ - geofabrik_url = self.o_geofabrik_json.find_geofbrik_url( + geofabrik_url = self.o_geofabrik_json.get_geofabrik_url( 'us/'+country.replace('_', '-')) skip_countries_built_url_is_false = [ @@ -142,6 +150,35 @@ def test_gofabrik_url_against_built_url(self): self.assertEqual(built_url, geofabrik_url, msg='country: '+country) + def test_reading_geofabrik_parent(self): + """ + go through all files in the wahoo_mc/resources/json directory + - compare the built URL via existing coding with the url out of the geofabrik json file + - some countries are skipped because they do not exist in geofabrik + - in addition + - skip countries if there is a geofabrik URL (because this URL is different to the built one) + - skip countries which do not have a direct geofabrik json entry, mostly they are in another country/region + """ + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'germany') + self.assertTrue(parent == 'europe' and child == 'germany') + + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'baden-wuerttemberg') + self.assertTrue(parent == 'germany' and child == 'baden-wuerttemberg') + + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'malta') + self.assertTrue(parent == 'europe' and child == 'malta') + + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'asia') + self.assertTrue(parent == '' and child == 'asia') + + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'xy') + self.assertTrue(parent is None and child is None) + def get_geofabrik_id_by_json_file_country(self, country): """ get geofabrik id by country .json filename @@ -154,13 +191,13 @@ def get_geofabrik_id_by_json_file_country(self, country): # get geofabrik id by country .json filename # 1. '_' replaced by '-' - child = self.o_geofabrik_json.find_geofbrik_parent( + child = self.o_geofabrik_json.get_geofabrik_parent_country( country.replace('_', '-'))[1] if child: return child # 2. 'us/' prefix and '_' replaced by '-' - child = self.o_geofabrik_json.find_geofbrik_parent( + child = self.o_geofabrik_json.get_geofabrik_parent_country( 'us/'+country.replace('_', '-'))[1] if child: diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 135e3afc..e47405f5 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -36,35 +36,58 @@ class GeofabrikJson: """ def __init__(self): + self.raw_json = [] + self.geofabrik_overview = {} + self.geofabrik_region_overview = {} with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: - self.json_data = geojson.load(file_handle) + self.raw_json = geojson.load(file_handle) file_handle.close() - def find_geofbrik_parent(self, name): + # create a dict with information easy to access because they are often needed + for feature in self.raw_json.features: + props = feature.properties + id_no = props['id'] + pbf_url = props['urls']['pbf'] + + try: + parent = props['parent'] + + self.geofabrik_overview[id_no] = { + 'parent': parent, + 'pbf_url': pbf_url} + except KeyError: + self.geofabrik_overview[id_no] = { + 'pbf_url': pbf_url} + self.geofabrik_region_overview[id_no] = { + 'pbf_url': pbf_url} + + print("done") + + def get_geofabrik_parent_country(self, id_no): """ Get the parent map/region of a region from the already loaded json data """ - for feature in self.json_data.features: - props = feature.properties - ident_no = props.get('id', '') - if ident_no != name: - continue - return (props.get('parent', ''), props.get('id', '')) - return None, None + try: + entry = self.geofabrik_overview[id_no] + if 'parent' in entry: + return (entry['parent'], id_no) + + return ('', id_no) + except KeyError: + return None, None - def find_geofbrik_url(self, name): + def get_geofabrik_url(self, id_no): """ Get the map download url from a region with the already loaded json data """ - for feature in self.json_data.features: - props = feature.properties - ident_no = props.get('id', '') - if ident_no != name: - continue - #print (props.get('urls', '')) - wurls = props.get('urls', '') - return wurls.get('pbf', '') + try: + entry = self.geofabrik_overview[id_no] + if 'pbf_url' in entry: + return entry['pbf_url'] + except KeyError: + pass + return None diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index 6bd6a426..995a3acf 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -197,7 +197,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): must_download_urls = [] # itterate through countries/regions in the geofabrik json file - for regions in o_geofabrik_json.json_data.features: + for regions in o_geofabrik_json.raw_json.features: props = regions.properties parent = props.get('parent', '') regionname = props.get('id', '') @@ -226,7 +226,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): x_value = 0 # handle sub-sub-regions like unterfranken->bayern->germany while parent not in geofabrik_regions: - parent, child = o_geofabrik_json.find_geofbrik_parent( + parent, child = o_geofabrik_json.get_geofabrik_parent_country( parent) if parent in geofabrik_regions: parent = child @@ -239,7 +239,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): if parent not in must_download_maps: must_download_maps.append(parent) must_download_urls.append( - o_geofabrik_json.find_geofbrik_url(parent)) + o_geofabrik_json.get_geofabrik_url(parent)) #parent_added = 1 else: if regionname not in must_download_maps: From 7ab38d9bb520a2f6e6af3476cd7fcf501df3d8fa Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Wed, 15 Feb 2023 23:08:20 +0100 Subject: [PATCH 07/21] Revert "variables are empty at that time" did break test_generated_files and test_osm_maps This reverts commit 03f31a263b0441849b3cbe626e5589346be049e0. --- wahoomc/osm_maps_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 329dc822..3a514ed1 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -148,7 +148,7 @@ def process_input_of_the_tool(self, o_input_data): """ o_downloader = Downloader( - o_input_data.max_days_old, o_input_data.force_download, None, None) + o_input_data.max_days_old, o_input_data.force_download, self.tiles, self.border_countries) # takeover what is given by user first for force_processing self.force_processing = o_input_data.force_processing From 02edba1b0f4106277d14e2ee8d6200b0eaccacab Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Wed, 15 Feb 2023 23:51:45 +0100 Subject: [PATCH 08/21] check input for valid geofabrik id --- tests/test_constants_geofabrik.py | 21 +++++++++++++++++++-- wahoomc/constants_functions.py | 9 +++++++++ wahoomc/input.py | 9 ++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index aec66f25..28e8c8dd 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -179,10 +179,27 @@ def test_reading_geofabrik_parent(self): 'xy') self.assertTrue(parent is None and child is None) + def test_if_input_is_geofabrik_id(self): + """ + check if the input is a id of the geofabrik file + """ + self.assertTrue( + self.o_geofabrik_json.is_input_a_geofabrik_id_no('germany')) + + self.assertTrue( + self.o_geofabrik_json.is_input_a_geofabrik_id_no('baden-wuerttemberg')) + + self.assertTrue( + self.o_geofabrik_json.is_input_a_geofabrik_id_no('asia')) + self.assertFalse( + self.o_geofabrik_json.is_input_a_geofabrik_id_no('xy')) + + self.assertFalse(self.o_geofabrik_json.is_input_a_geofabrik_id_no('*')) + def get_geofabrik_id_by_json_file_country(self, country): """ - get geofabrik id by country .json filename - """ + get geofabrik id by country .json filename + """ # get geofabrik id by country .json filename # 1. raw # parent, child = self.o_geofabrik_json.find_geofbrik_parent( diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index e47405f5..bc6da2ef 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -90,6 +90,15 @@ def get_geofabrik_url(self, id_no): return None + def is_input_a_geofabrik_id_no(self, id_no): + """ + check if the given input is a geofabrik id number + """ + if id_no in self.geofabrik_overview: + return True + + return False + def get_region_of_country(county): """ diff --git a/wahoomc/input.py b/wahoomc/input.py index 6e84cca3..e698eb4f 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -14,6 +14,9 @@ # import custom python packages from wahoomc import constants +from wahoomc.constants_functions import GeofabrikJson + +o_geofabrik_json = GeofabrikJson() def process_call_of_the_tool(): @@ -194,6 +197,10 @@ def is_required_input_given_or_exit(self, issue_message): elif self.country and self.xy_coordinates: sys.exit( "Country and X/Y coordinates are given. Only one of both is allowed!") + elif self.country and not o_geofabrik_json.is_input_a_geofabrik_id_no(self.country): + sys.exit( + f"Entered country '{self.country}' is not a geofabrik country. Please check this URL for possible countries \ + https://download.geofabrik.de/index.html!") else: return True @@ -433,4 +440,4 @@ def __init__(self, parent, oInputData): self.checkb_zip_folder_val = create_checkbox(self, oInputData.zip_folder, "Zip folder with generated files", 3) self.checkb_verbose_val = create_checkbox(self, oInputData.verbose, - "output debug logger messages", 4) + "output debug logger messages", 4) From 3cd26349e2667040511237e8242936483e6024b7 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Fri, 17 Feb 2023 05:50:40 +0100 Subject: [PATCH 09/21] unittest: compare regions --- tests/test_constants_geofabrik.py | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 28e8c8dd..5f3302d4 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -10,6 +10,7 @@ from wahoomc.downloader import get_osm_pbf_filepath_url from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson +from wahoomc.constants_functions import get_geofabrik_region_of_country # json countries with no geofabrik id partner json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', @@ -18,6 +19,13 @@ 'norfolk_island', 'wallis_and_futuna', 'northern_mariana_islands', 'paracel_islands', 'united_arab_emirates', 'kuwait', 'qatar', 'spratly_islands', 'singapore', 'brunei', 'bahrain', 'macao', 'cocos_islands', 'christmas_island', 'palestina', 'malaysia', 'saudi_arabia', 'british_indian_ocean_territory', 'israel', 'oman', 'hong_kong', 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', 'guam', 'commonwealth_of_the_northern_mariana_islands', 'american_samoa', 'united_states_virgin_islands', 'svalbard_and_jan_mayen', 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'vatican_city', 'ireland', 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'montserrat', 'bermuda', 'virgin_islands_u.s.', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', 'anguilla', 'saint-martin', 'cayman_islands', 'sint_maarten', 'haiti', 'saint_lucia', 'british_virgin_islands', 'saint_kitts_and_nevis', 'dominican_republic', 'turks_and_caicos_islands', 'antigua_and_barbuda', 'gambia', 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'comoros', 'republic_of_congo', 'democratic_republic_of_the_congo', 'senegal', 'french_southern_territories'] +# json countries with no geofabrik region defined in constants.py +no_geofabrik_region_assigned = ['guyana', 'solomon_islands', 'marshall_islands', 'pitcairn_islands', 'cook_islands', 'new_caledonia', + 'new_zealand', 'alaska', 'oklahoma', 'puerto_rico', 'oregon', 'panama', + 'martinique', 'costa_rica', 'guadeloupe', 'el_salvador', 'mayotte', + 'sierra_leone', 'central_african_republic', 'sao_tome_and_principe', + 'equatorial_guinea', 'reunion'] + class TestConstantsGeofabrik(unittest.TestCase): """ @@ -82,6 +90,43 @@ def test_regions_of_geofabrik(self): self.assertCountEqual(geofabrik_regions_w_russia, regions_geofabrik) + def test_geofabrik_region_against_get_region(self): + """ + go through all files in the wahoo_mc/resources/json directory + - compare the region from get_geofabrik_region_of_country against the geofabrik json file + - some countries are skipped because + - there is no get_geofabrik_region_of_country region + - get_geofabrik_region_of_country returns false region + """ + for country in self.relevant_countries: + get_region = '' + parent = '' + + try: + get_region = get_geofabrik_region_of_country(country) + except SystemExit: + # if there is no get_region, everything else is better ;-) + self.assertIn(country, no_geofabrik_region_assigned) + continue + + id_no = self.get_geofabrik_id_by_json_file_country(country) + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + id_no) + + # these are own continents without region/parent. + if country in ['antarctica', 'russia']: + self.assertEqual(parent, '') + # these get false routed via get_region. compare against correct parent + elif country == 'papua_new_guinea': + self.assertEqual(parent, 'australia-oceania') + elif country == 'georgia': + self.assertEqual(parent, 'europe') + else: + # "normal" check + # one geofabrik region in constsants.py has north-america/us + self.assertTrue( + get_region in [parent, parent + '/us'], 'country: '+country) + def test_gofabrik_url_against_built_url(self): """ go through all files in the wahoo_mc/resources/json directory From 3f20d9b49ea61eb7060bd2f2ab99221de19b1bec Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Fri, 17 Feb 2023 14:17:47 +0100 Subject: [PATCH 10/21] unittests make no sense - tiles X/Y are mostly different - latitude longitude are different even if X/Y are equal --- tests/test_geofabrik.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/test_geofabrik.py b/tests/test_geofabrik.py index e168ec44..40446bca 100644 --- a/tests/test_geofabrik.py +++ b/tests/test_geofabrik.py @@ -7,8 +7,6 @@ # import custom python packages from wahoomc.geofabrik import Geofabrik -from wahoomc import file_directory_functions as fd_fct -from wahoomc import constants_functions as const_fct from wahoomc.downloader import Downloader from wahoomc import constants @@ -24,18 +22,6 @@ def calc_tiles_via_geofabrik_json(input_argument): return tiles_via_geofabrik_json -def calc_tiles_via_static_jsons(input_argument): - """ - calculate tiles using the json files in the repo - the "old" way of doing - """ - json_file_path = os.path.join(constants.RESOURCES_DIR, 'json', - const_fct.get_region_of_country(input_argument), input_argument + '.json') - tiles_via_static_json = fd_fct.read_json_file(json_file_path) - - return tiles_via_static_json - - class TestGeofabrik(unittest.TestCase): """ tests for the downloader file @@ -66,30 +52,6 @@ def test_tiles_via_geofabrik_malta(self): self.assertEqual(item_0, geofabrik_tiles[0]) self.assertEqual(item_1, geofabrik_tiles[1]) - # def test_tiles_via_url_germany(self): - # """ - # Test the retrieval of tiles via geofabrik URL against hardcoded data for germany - # """ - # self.compare_url_and_static('germany') - - # def test_tiles_via_url_ireland(self): - # """ - # Test the retrieval of tiles via URL - # """ - # self.compare_url_and_static('ireland-and-northern-ireland') - - def compare_geofabrik_and_static(self, input_argument): - """ - Compare the retrieval of tiles via URL with statis json - """ - tiles_via_geofabrik_json = calc_tiles_via_geofabrik_json( - input_argument) - - tiles_via_static_json = calc_tiles_via_static_jsons( - input_argument) - - self.assertEqual(tiles_via_geofabrik_json, tiles_via_static_json) - if __name__ == '__main__': unittest.main() From 42be789f7a103cb7b829951a507544181ddc816e Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Fri, 17 Feb 2023 15:35:19 +0100 Subject: [PATCH 11/21] only read geofabrik .json file once not per each instantiation --- wahoomc/constants_functions.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index bc6da2ef..439c9f7f 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -34,18 +34,30 @@ class GeofabrikJson: """ This is a Geofabrik .json processing class for constants in the Geofabrik .json file """ + raw_json = None + geofabrik_overview = {} + geofabrik_region_overview = {} def __init__(self): - self.raw_json = [] - self.geofabrik_overview = {} - self.geofabrik_region_overview = {} + # read geofabrik .json file and fill class-attributes // access is only once + if GeofabrikJson.raw_json is None and not GeofabrikJson.geofabrik_overview \ + and not GeofabrikJson.geofabrik_region_overview: + GeofabrikJson.raw_json, GeofabrikJson.geofabrik_overview, GeofabrikJson.geofabrik_region_overview = self.read_geofabrik_json_file() + + def read_geofabrik_json_file(self): + """ + read geofabrik .json file and fill class-attributes + """ + raw_json = [] + geofabrik_overview = {} + geofabrik_region_overview = {} with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: - self.raw_json = geojson.load(file_handle) + raw_json = geojson.load(file_handle) file_handle.close() # create a dict with information easy to access because they are often needed - for feature in self.raw_json.features: + for feature in raw_json.features: props = feature.properties id_no = props['id'] pbf_url = props['urls']['pbf'] @@ -53,16 +65,16 @@ def __init__(self): try: parent = props['parent'] - self.geofabrik_overview[id_no] = { + geofabrik_overview[id_no] = { 'parent': parent, 'pbf_url': pbf_url} except KeyError: - self.geofabrik_overview[id_no] = { + geofabrik_overview[id_no] = { 'pbf_url': pbf_url} - self.geofabrik_region_overview[id_no] = { + geofabrik_region_overview[id_no] = { 'pbf_url': pbf_url} - print("done") + return raw_json, geofabrik_overview, geofabrik_region_overview def get_geofabrik_parent_country(self, id_no): """ From a9f5aea7a1a4c463a9d759c9de0442b2cd64f09e Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Fri, 17 Feb 2023 21:34:30 +0100 Subject: [PATCH 12/21] fetch geofabrik URL instead of composing - get_osm_pbf_filepath_url might be fully removed later on - translating input to valid geofabrik ident no in class GeofabrikJson - return of get_geofabrik_parent changed --> adjust test_reading_geofabrik_parent --- tests/test_constants_geofabrik.py | 8 ++++---- wahoomc/constants_functions.py | 32 +++++++++++++++++++++++++++---- wahoomc/downloader.py | 21 ++++++++------------ 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 5f3302d4..83964f0c 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -10,7 +10,7 @@ from wahoomc.downloader import get_osm_pbf_filepath_url from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson -from wahoomc.constants_functions import get_geofabrik_region_of_country +from wahoomc.constants_functions import get_geofabrik_region_of_country, CountyIsNoGeofabrikCountry # json countries with no geofabrik id partner json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', @@ -220,9 +220,9 @@ def test_reading_geofabrik_parent(self): 'asia') self.assertTrue(parent == '' and child == 'asia') - parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( - 'xy') - self.assertTrue(parent is None and child is None) + with self.assertRaises(CountyIsNoGeofabrikCountry): + parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( + 'xy') def test_if_input_is_geofabrik_id(self): """ diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 439c9f7f..ad99017f 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -30,6 +30,10 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" +class CountyIsNoGeofabrikCountry(Exception): + """Raised when actual country is not a geofabrik country""" + + class GeofabrikJson: """ This is a Geofabrik .json processing class for constants in the Geofabrik .json file @@ -80,12 +84,14 @@ def get_geofabrik_parent_country(self, id_no): """ Get the parent map/region of a region from the already loaded json data """ + id_no_translated = self.translate_id_no_to_geofabrik(id_no) + try: - entry = self.geofabrik_overview[id_no] + entry = self.geofabrik_overview[id_no_translated] if 'parent' in entry: - return (entry['parent'], id_no) + return (entry['parent'], id_no_translated) - return ('', id_no) + return ('', id_no_translated) except KeyError: return None, None @@ -93,8 +99,9 @@ def get_geofabrik_url(self, id_no): """ Get the map download url from a region with the already loaded json data """ + id_no_translated = self.translate_id_no_to_geofabrik(id_no) try: - entry = self.geofabrik_overview[id_no] + entry = self.geofabrik_overview[id_no_translated] if 'pbf_url' in entry: return entry['pbf_url'] except KeyError: @@ -111,6 +118,23 @@ def is_input_a_geofabrik_id_no(self, id_no): return False + def translate_id_no_to_geofabrik(self, country): + """ + get geofabrik id by country .json filename + """ + + if country in self.geofabrik_overview: + return country + + if country.replace('_', '-') in self.geofabrik_overview: + return country.replace('_', '-') + + if 'us/'+country.replace('_', '-') in self.geofabrik_overview: + return 'us/'+country.replace('_', '-') + + # if none of them got triggert --> exception + raise CountyIsNoGeofabrikCountry + def get_region_of_country(county): """ diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index fca63ac2..85b12686 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -15,8 +15,8 @@ # import custom python packages from wahoomc.file_directory_functions import download_url_to_file, unzip -from wahoomc.constants_functions import translate_country_input_to_geofabrik, \ - get_geofabrik_region_of_country, get_tooling_win_path +from wahoomc.constants_functions import translate_country_input_to_geofabrik, get_tooling_win_path +from wahoomc.constants_functions import GeofabrikJson from wahoomc.constants import USER_DL_DIR from wahoomc.constants import USER_MAPS_DIR @@ -75,22 +75,17 @@ def download_file(target_filepath, url, target_dir=""): def get_osm_pbf_filepath_url(country): """ - build the geofabrik download url to a countries' OSM file and download filepath + fetch the geofabrik download url to countries' OSM file + and build download filepath to countries' OSM file """ + o_geofabrik_json = GeofabrikJson() - transl_c = translate_country_input_to_geofabrik(country) - region = get_geofabrik_region_of_country(country) - if region != 'no': - url = 'https://download.geofabrik.de/' + region + \ - '/' + transl_c + '-latest.osm.pbf' - else: - url = 'https://download.geofabrik.de/' + transl_c + '-latest.osm.pbf' - + # build path to downloaded file with translated geofabrik country map_file_path = os.path.join( - USER_MAPS_DIR, f'{transl_c}' + '-latest.osm.pbf') + USER_MAPS_DIR, f'{o_geofabrik_json.get_geofabrik_parent(country)[1]}' + '-latest.osm.pbf') # return URL and download filepath - return map_file_path, url + return map_file_path, o_geofabrik_json.get_geofabrik_url(country) def download_tooling(): From 89a03672e01514192b0c371c77ffded91203ceca Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 05:56:23 +0100 Subject: [PATCH 13/21] reduce function to make it readabler - comparing via test_gofabrik_url_against_built_url makes no sense anymore --- tests/test_constants_geofabrik.py | 69 ------------------------------- tests/test_downloader.py | 8 +++- wahoomc/downloader.py | 22 ++++++---- 3 files changed, 19 insertions(+), 80 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 83964f0c..965236d6 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -7,7 +7,6 @@ # import custom python packages from wahoomc import file_directory_functions as fd_fct from wahoomc.downloader import Downloader -from wahoomc.downloader import get_osm_pbf_filepath_url from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson from wahoomc.constants_functions import get_geofabrik_region_of_country, CountyIsNoGeofabrikCountry @@ -127,74 +126,6 @@ def test_geofabrik_region_against_get_region(self): self.assertTrue( get_region in [parent, parent + '/us'], 'country: '+country) - def test_gofabrik_url_against_built_url(self): - """ - go through all files in the wahoo_mc/resources/json directory - - compare the built URL via existing coding with the url out of the geofabrik json file - - some countries are skipped because they do not exist in geofabrik - - in addition - - skip countries if there is a geofabrik URL (because this URL is different to the built one) - - skip countries which do not have a direct geofabrik json entry, mostly they are in another country/region - """ - for country in self.relevant_countries: - built_url = None - geofabrik_url = None - - # build URL with existing coding - try: - map_file_path, built_url = get_osm_pbf_filepath_url(country) - except SystemExit: - pass - - # grab URL from geofabrik json - geofabrik_url = self.o_geofabrik_json.get_geofabrik_url(country) - - if not geofabrik_url: - # check with replaced _ by - - geofabrik_url = self.o_geofabrik_json.get_geofabrik_url( - country.replace('_', '-')) - if not geofabrik_url: - # check with replaced _ by - AND prefix us/ - geofabrik_url = self.o_geofabrik_json.get_geofabrik_url( - 'us/'+country.replace('_', '-')) - - skip_countries_built_url_is_false = [ - 'papua_new_guinea', 'east_timor', - 'antarctica', # built url is false - 'georgia' # will issue north-america/us but europe is right. Reason: order in get_geofabrik_region_of_country - ] - - # skip_countries_for_url_comparison = [ - # 'united_arab_emirates', 'kuwait', 'qatar', 'bahrain', 'saudi_arabia', 'oman', # gcc-states - # 'malaysia', 'singapore', 'brunei', # malaysia-singapore-brunei - # 'christmas_island', # indonesia - # 'paracel_islands', 'spratly_islands', 'british_indian_ocean_territory', # asia - # 'macao', 'hong_kong', # china - # 'palestina', 'israel', # israel-and-palestine - # 'northern_mariana_islands', 'cocos_islands', 'commonwealth_of_the_northern_mariana_islands', 'american_samoa', # australia - # 'antarctica', # built url is false - # 'georgia', # will issue north-america/us but europe is right because of the order in get_geofabrik_region_of_country - # 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'san_marino', 'vatican_city', 'ireland', # europe - # 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'svalbard_and_jan_mayen', # europe - # 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'democratic_republic_of_the_congo'] # africa - - if built_url != geofabrik_url: - # # either it is a country to skip - # if (country in skip_countries_for_url_comparison_if_geofabrik_url and geofabrik_url) \ - # or country in skip_countries_for_url_comparison: - # continue - - # or there is no built url but a geofabrik one - ok! - # for some countries the built url is false - if built_url is None or \ - country in skip_countries_built_url_is_false: - self.assertTrue(geofabrik_url, msg='country: '+country) - continue - - # "normal" compare, also if both are empty/ None - self.assertEqual(built_url, geofabrik_url, - msg='country: '+country) - def test_reading_geofabrik_parent(self): """ go through all files in the wahoo_mc/resources/json directory diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 0a0d0ed5..42cbeb4f 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -13,11 +13,12 @@ from wahoomc.downloader import older_than_x_days from wahoomc.downloader import download_file -from wahoomc.downloader import get_osm_pbf_filepath_url +from wahoomc.downloader import build_osm_pbf_filepath from wahoomc.downloader import download_tooling from wahoomc.downloader import Downloader from wahoomc import constants from wahoomc.constants_functions import get_tooling_win_path +from wahoomc.constants_functions import GeofabrikJson class TestDownloader(unittest.TestCase): @@ -116,10 +117,13 @@ def test_download_malta_osm_pbf_file(self): path = os.path.join(constants.USER_DL_DIR, 'maps', f'{country}' + '-latest.osm.pbf') + o_geofabrik_json = GeofabrikJson() + if os.path.exists(path): os.remove(path) - map_file_path, url = get_osm_pbf_filepath_url(country) + map_file_path = build_osm_pbf_filepath(country) + url = o_geofabrik_json.get_geofabrik_url(country) download_file(map_file_path, url, False) self.assertTrue(os.path.exists(path)) diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index 85b12686..a056ed03 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -73,19 +73,17 @@ def download_file(target_filepath, url, target_dir=""): log.info('+ Downloaded: %s', target_filepath) -def get_osm_pbf_filepath_url(country): +def build_osm_pbf_filepath(country_translated): """ - fetch the geofabrik download url to countries' OSM file - and build download filepath to countries' OSM file + build download filepath to countries' OSM file + replace / to have no problem with directories """ - o_geofabrik_json = GeofabrikJson() - # build path to downloaded file with translated geofabrik country map_file_path = os.path.join( - USER_MAPS_DIR, f'{o_geofabrik_json.get_geofabrik_parent(country)[1]}' + '-latest.osm.pbf') + USER_MAPS_DIR, f'{country_translated.replace("/", "_")}' + '-latest.osm.pbf') - # return URL and download filepath - return map_file_path, o_geofabrik_json.get_geofabrik_url(country) + # return download filepath + return map_file_path def download_tooling(): @@ -289,10 +287,16 @@ def download_osm_pbf_file(self): """ download countries' OSM files """ + o_geofabrik_json = GeofabrikJson() + for country, item in self.border_countries.items(): try: if item['download'] is True: - map_file_path, url = get_osm_pbf_filepath_url(country) + # build path to downloaded file with translated geofabrik country + map_file_path = build_osm_pbf_filepath( + o_geofabrik_json.translate_id_no_to_geofabrik(country)) + # fetch the geofabrik download url to countries' OSM file + url = o_geofabrik_json.get_geofabrik_url(country) download_file(map_file_path, url) self.border_countries[country] = { 'map_file': map_file_path} From f61a89f90fbf3ff9361f7af6bb0d54e3a1c4d30a Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 06:20:18 +0100 Subject: [PATCH 14/21] additional unittests --- tests/test_constants_geofabrik.py | 16 ++++++++ tests/test_downloader.py | 62 +++++++++++++++++++++++++------ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 965236d6..1cd44e43 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -172,6 +172,22 @@ def test_if_input_is_geofabrik_id(self): self.assertFalse(self.o_geofabrik_json.is_input_a_geofabrik_id_no('*')) + def test_tranlation_id_to_geofabrik_id(self): + """ + get geofabrik id by country .json filename + """ + transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik( + 'malta') + self.assertEqual('malta', transl_country) + + transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik( + 'solomon_islands') + self.assertEqual('solomon-islands', transl_country) + + transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik( + 'nebraska') + self.assertEqual('us/nebraska', transl_country) + def get_geofabrik_id_by_json_file_country(self, country): """ get geofabrik id by country .json filename diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 42cbeb4f..44cd8dbf 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -107,26 +107,38 @@ def test_download_geofabrik_file_2(self): self.assertFalse( self.o_downloader.should_geofabrik_file_be_downloaded()) - def test_download_malta_osm_pbf_file(self): + def test_building_osm_pbf_filepaths(self): """ - Test the download of geofabrik file via URL + Test if composing map file path on the device is correct + "/" to "_" is done for "us/" countries """ + self.check_exp_agains_composed_map_file_path( + 'us/nebraska', 'us_nebraska') - country = 'malta' + self.check_exp_agains_composed_map_file_path( + 'us/georgia', 'us_georgia') - path = os.path.join(constants.USER_DL_DIR, 'maps', - f'{country}' + '-latest.osm.pbf') + self.check_exp_agains_composed_map_file_path( + 'malta', 'malta') - o_geofabrik_json = GeofabrikJson() + self.check_exp_agains_composed_map_file_path( + 'asia', 'asia') - if os.path.exists(path): - os.remove(path) + self.check_exp_agains_composed_map_file_path( + 'georgia', 'georgia') - map_file_path = build_osm_pbf_filepath(country) - url = o_geofabrik_json.get_geofabrik_url(country) - download_file(map_file_path, url, False) + def test_download_osm_pbf_file(self): + """ + Test the download of geofabrik file via URL + """ + self.check_download_osm_pbf_file('malta', os.path.join(constants.USER_DL_DIR, 'maps', + 'malta' + '-latest.osm.pbf')) - self.assertTrue(os.path.exists(path)) + self.check_download_osm_pbf_file('us/hawaii', os.path.join(constants.USER_DL_DIR, 'maps', + 'us_hawaii' + '-latest.osm.pbf')) + + self.check_download_osm_pbf_file('solomon-islands', os.path.join(constants.USER_DL_DIR, 'maps', + 'solomon-islands' + '-latest.osm.pbf')) def test_delete_not_existing_file(self): """ @@ -193,6 +205,32 @@ def test_download_macos_files(self): self.assertTrue(os.path.exists(path)) + def check_exp_agains_composed_map_file_path(self, country, country_map_file_path): + """ + helper function to check a given to-be-path against the composed on based on country + """ + exp_path = os.path.join(constants.USER_DL_DIR, 'maps', + f'{country_map_file_path}' + '-latest.osm.pbf') + + composed_path = build_osm_pbf_filepath(country) + + self.assertEqual(exp_path, composed_path) + + def check_download_osm_pbf_file(self, country, path): + """ + Test if the download of a input country created the wanted map file + """ + o_geofabrik_json = GeofabrikJson() + + if os.path.exists(path): + os.remove(path) + + map_file_path = build_osm_pbf_filepath(country) + url = o_geofabrik_json.get_geofabrik_url(country) + download_file(map_file_path, url, False) + + self.assertTrue(os.path.exists(path)) + if __name__ == '__main__': unittest.main() From 6bd51dac9be96c2a01988a8a097b9656df007347 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 06:42:45 +0100 Subject: [PATCH 15/21] remove unnused functions and constants - superseeded by fully relying on geofabrik --- tests/test_constants.py | 34 +------ tests/test_constants_geofabrik.py | 46 +--------- wahoomc/constants.py | 144 ------------------------------ wahoomc/constants_functions.py | 54 ----------- wahoomc/downloader.py | 13 +-- 5 files changed, 9 insertions(+), 282 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 0ee6a2e4..184e79b3 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -6,43 +6,11 @@ import mock -from wahoomc.constants_functions import translate_country_input_to_geofabrik, translate_tags_to_keep, \ +from wahoomc.constants_functions import translate_tags_to_keep, \ get_tag_wahoo_xml_path, TagWahooXmlNotFoundError from wahoomc.constants import RESOURCES_DIR -class TestTranslateCountries(unittest.TestCase): - """ - tests for translating country-constants between geofabrik and other formats - """ - - def test_translated_countries_to_china(self): - """ - Test countries which have no own geofabrik country but are included in china - """ - - expected = 'china' - - transl_c = translate_country_input_to_geofabrik('hong_kong') - self.assertEqual(expected, transl_c) - - transl_c = translate_country_input_to_geofabrik('macao') - self.assertEqual(expected, transl_c) - - transl_c = translate_country_input_to_geofabrik('paracel_islands') - self.assertEqual(expected, transl_c) - - def test_translated_countries_no_mapping(self): - """ - Test countries which have no own geofabrik country but are included in china - """ - - expected = 'germany' - - transl_c = translate_country_input_to_geofabrik('germany') - self.assertEqual(expected, transl_c) - - tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": { 'access': '', 'area': 'yes' diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 1cd44e43..b846ed7f 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -9,7 +9,7 @@ from wahoomc.downloader import Downloader from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson -from wahoomc.constants_functions import get_geofabrik_region_of_country, CountyIsNoGeofabrikCountry +from wahoomc.constants_functions import CountyIsNoGeofabrikCountry # json countries with no geofabrik id partner json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', @@ -18,13 +18,6 @@ 'norfolk_island', 'wallis_and_futuna', 'northern_mariana_islands', 'paracel_islands', 'united_arab_emirates', 'kuwait', 'qatar', 'spratly_islands', 'singapore', 'brunei', 'bahrain', 'macao', 'cocos_islands', 'christmas_island', 'palestina', 'malaysia', 'saudi_arabia', 'british_indian_ocean_territory', 'israel', 'oman', 'hong_kong', 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', 'guam', 'commonwealth_of_the_northern_mariana_islands', 'american_samoa', 'united_states_virgin_islands', 'svalbard_and_jan_mayen', 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'vatican_city', 'ireland', 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'montserrat', 'bermuda', 'virgin_islands_u.s.', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', 'anguilla', 'saint-martin', 'cayman_islands', 'sint_maarten', 'haiti', 'saint_lucia', 'british_virgin_islands', 'saint_kitts_and_nevis', 'dominican_republic', 'turks_and_caicos_islands', 'antigua_and_barbuda', 'gambia', 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'comoros', 'republic_of_congo', 'democratic_republic_of_the_congo', 'senegal', 'french_southern_territories'] -# json countries with no geofabrik region defined in constants.py -no_geofabrik_region_assigned = ['guyana', 'solomon_islands', 'marshall_islands', 'pitcairn_islands', 'cook_islands', 'new_caledonia', - 'new_zealand', 'alaska', 'oklahoma', 'puerto_rico', 'oregon', 'panama', - 'martinique', 'costa_rica', 'guadeloupe', 'el_salvador', 'mayotte', - 'sierra_leone', 'central_african_republic', 'sao_tome_and_principe', - 'equatorial_guinea', 'reunion'] - class TestConstantsGeofabrik(unittest.TestCase): """ @@ -89,43 +82,6 @@ def test_regions_of_geofabrik(self): self.assertCountEqual(geofabrik_regions_w_russia, regions_geofabrik) - def test_geofabrik_region_against_get_region(self): - """ - go through all files in the wahoo_mc/resources/json directory - - compare the region from get_geofabrik_region_of_country against the geofabrik json file - - some countries are skipped because - - there is no get_geofabrik_region_of_country region - - get_geofabrik_region_of_country returns false region - """ - for country in self.relevant_countries: - get_region = '' - parent = '' - - try: - get_region = get_geofabrik_region_of_country(country) - except SystemExit: - # if there is no get_region, everything else is better ;-) - self.assertIn(country, no_geofabrik_region_assigned) - continue - - id_no = self.get_geofabrik_id_by_json_file_country(country) - parent, child = self.o_geofabrik_json.get_geofabrik_parent_country( - id_no) - - # these are own continents without region/parent. - if country in ['antarctica', 'russia']: - self.assertEqual(parent, '') - # these get false routed via get_region. compare against correct parent - elif country == 'papua_new_guinea': - self.assertEqual(parent, 'australia-oceania') - elif country == 'georgia': - self.assertEqual(parent, 'europe') - else: - # "normal" check - # one geofabrik region in constsants.py has north-america/us - self.assertTrue( - get_region in [parent, parent + '/us'], 'country: '+country) - def test_reading_geofabrik_parent(self): """ go through all files in the wahoo_mc/resources/json directory diff --git a/wahoomc/constants.py b/wahoomc/constants.py index 6e995f27..16bf6c66 100644 --- a/wahoomc/constants.py +++ b/wahoomc/constants.py @@ -27,76 +27,6 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) VERSION = '3.2.0' -Translate_Country = { - 'alaska': 'united_states_of_america', - 'anguilla': 'central-america', - 'bahrain': 'gcc-states', - 'bosnia_and_herzegovina': 'bosnia-herzegovina', - 'british_virgin_islands': 'central-america', - 'british_indian_ocean_territory': 'asia', - 'brunei': 'malaysia-singapore-brunei', - 'burkina_faso': 'burkina-faso', - 'canary_islands': 'canary-islands', - 'cape_verde': 'cape-verde', - 'christmas_island': 'indonesia', - 'cocos_islands': 'australia', - 'cote_d_ivoire': 'ivory-coast', - 'czech_republic': 'czech-republic', - 'democratic_republic_of_the_congo': 'congo-democratic-republic', - 'east_timor': 'indonesia', - 'faroe_islands': 'faroe-islands', - 'gambia': 'senegal-and-gambia', - 'gibraltar': 'spain', - 'guernsey': 'guernsey-jersey', - 'hong_kong': 'china', - 'ireland': 'ireland-and-northern-ireland', - 'isle_of_man': 'isle-of-man', - 'israel': 'israel-and-palestine', - 'jersey': 'guernsey-jersey', - 'kuwait': 'gcc-states', - 'macao': 'china', - 'malaysia': 'malaysia-singapore-brunei', - 'north_korea': 'north-korea', - 'oman': 'gcc-states', - 'palestina': 'israel-and-palestine', - 'papua_new_guinea': 'asia', - 'paracel_islands': 'china', - 'republic_of_congo': 'congo-brazzaville', - 'saint-martin': 'central-america', - 'saint_helena': 'saint-helena-ascension-and-tristan-da-cunha', - 'san_marino': 'italy', - 'saudi_arabia': 'gcc-states', - 'senegal': 'senegal-and-gambia', - 'singapore': 'malaysia-singapore-brunei', - 'sint_maarten': 'central-america', - 'south_africa': 'south-africa', - 'south_korea': 'south-korea', - 'spratly_islands': 'asia', - 'sri_lanka': 'sri-lanka', - 'svalbard_and_jan_mayen': 'norway', - 'united_arab_emirates': 'gcc-states', - 'united_kingdom': 'great-britain', - 'united_states_virgin_islands': 'central-america', - 'vatican_city': 'italy', - 'virgin_islands_u.s.': 'central-america', - 'western_sahara': 'morocco', - 'qatar': 'gcc-states', - 'åland': 'finland', - 'new_mexico': 'new-mexico', - 'american_samoa': 'samoa', - 'commonwealth_of_the_northern_mariana_islands': 'american-oceania', - 'northern_mariana_islands': 'american-oceania', - 'new_york': 'new-york', - 'new_hampshire': 'new-hampshire', - 'new_jersey': 'new-jersey', - 'rhode_island': 'rhode-island', - 'district_of_columbia': 'district-of-columbia', - 'north_carolina': 'north-carolina', - 'south_carolina': 'south-carolina', - 'north_dakota': 'north-dakota', - 'south_dakota': 'south-dakota', - 'west_virginia': 'west-virginia' -} continents = ['europe', 'unitedstates', 'north-america', 'south-america', 'asia', 'oceania', 'africa', 'antarctica'] @@ -166,80 +96,6 @@ 'south_dakota', 'tennessee', 'texas', 'united_states_virgin_islands', 'utah', 'vermont', 'virginia', 'washington', 'west_virginia', 'wisconsin', 'wyoming'] -africa_geofabrik = ['algeria', 'angola', 'benin', 'botswana', 'burkina-faso', 'burundi', - 'cameroon', 'canary-islands', 'cape-verde', 'central african republic', 'chad', - 'comores', 'Congo (Republic/Brazzaville)', 'congo-democratic-republic', 'djibouti', - 'egypt', 'Equatorial Guinea', 'eritrea', 'ethiopia', 'gabon', 'ghana', 'guinea', - 'guinea-bissau', 'ivory-coast', 'kenya', 'lesotho', 'liberia', 'libya', 'madagascar', - 'malawi', 'mali', 'mauritania', 'mauritius', 'morocco', 'mozambique', 'namibia', - 'niger', 'nigeria', 'rwanda', 'saint-helena-ascension-and-tristan-da-cunha', - 'Sao Tome and Principe', 'Senegal and Gambia', 'seychelles', 'Sierra Leone', - 'somalia', 'south-africa', 'South Sudan', 'sudan', 'swaziland', 'tanzania', 'togo', - 'tunisia', 'uganda', 'zambia', 'zimbabwe'] - -antarctica_geofabrik = ['antarctica'] - -asia_geofabrik = ['afghanistan', 'armenia', 'azerbaijan', 'bangladesh', 'bhutan', 'cambodia', - 'china', 'gcc-states', 'india', 'indonesia', 'iran', 'iraq', 'israel-and-palestine', - 'japan', 'jordan', 'kazakhstan', 'kyrgyzstan', 'laos', 'lebanon', - 'malaysia-singapore-brunei', 'maldives', 'mongolia', 'myanmar', 'nepal', 'north-korea', - 'pakistan', 'philippines', 'russian federation', 'south-korea', 'sri-lanka', 'syria', - 'taiwan', 'tajikistan', 'thailand', 'turkmenistan', 'uzbekistan', 'vietnam', 'yemen'] - -australiaoceania_geofabrik = ['american-oceania', 'australia', 'cook islands', 'fiji', - 'île de clipperton', 'kiribati', 'marshall islands', 'micronesia', 'nauru', - 'new caledonia', 'new zealand', 'niue', 'palau', 'papua new guinea', - 'pitcairn islands', 'polynesie-francaise', 'samoa', 'solomon islands', 'tokelau', - 'tonga', 'tuvalu', 'vanuatu', 'wallis et futuna'] - -centralamerica_geofabrik = ['bahamas', 'belize', 'costa rica', 'cuba', 'el salvador', 'guatemala', - 'haiti and dominican republic', 'honduras', 'jamaica', 'nicaragua'] - -europe_geofabrik = ['albania', 'andorra', 'austria', 'azores', 'belarus', 'belgium', - 'bosnia-herzegovina', 'bulgaria', 'croatia', 'cyprus', 'czech-republic', 'denmark', - 'estonia', 'faroe-islands', 'finland', 'france', 'georgia', 'germany', 'great-britain', - 'greece', 'guernsey-jersey', 'hungary', 'iceland', 'ireland-and-northern-ireland', - 'isle-of-man', 'italy', 'kosovo', 'latvia', 'liechtenstein', 'lithuania', 'luxembourg', - 'macedonia', 'malta', 'moldova', 'monaco', 'montenegro', 'netherlands', 'norway', - 'poland', 'portugal', 'romania', 'russian federation', 'serbia', 'slovakia', - 'slovenia', 'spain', 'sweden', 'switzerland', 'turkey', 'ukraine'] - -northamerica_geofabrik = ['canada', 'greenland', 'mexico', 'us midwest', 'us northeast', - 'us pacific', 'us south', 'us west'] - -northamerica_us_geofabrik = ['alabama', 'alaska', 'arizona', 'arkansas', 'california', 'colorado', - 'connecticut', 'delaware', 'district-of-columbia', 'florida', - 'georgia', 'hawaii', 'idaho', 'illinois', 'indiana', 'iowa', 'kansas', - 'kentucky', 'louisiana', 'maine', 'maryland', 'massachusetts', - 'michigan', 'minnesota', 'mississippi', 'missouri', 'montana', - 'nebraska', 'nevada', 'new-hampshire', 'new-jersey', 'new-mexico', - 'new-york', 'north-carolina', 'north-dakota', 'ohio', 'oklahoma' - 'oregon', 'pennsylvania', 'puerto-rico', 'rhode-island', - 'south-carolina', 'south-dakota', 'tennessee', 'texas', - 'us-virgin-islands', 'utah', 'vermont', 'virginia', 'washington', - 'west-virginia', 'wisconsin', 'wyoming'] - -southamerica_geofabrik = ['argentina', 'bolivia', 'brazil', 'chile', 'colombia', 'ecuador', - 'paraguay', 'peru', 'suriname', 'uruguay', 'venezuela'] - -germany_subregions_geofabrik = ['baden-württemberg', 'bayern', 'berlin', - 'brandenburg (mit berlin)', 'bremen', 'hamburg', 'hessen', 'mecklenburg-vorpommern', - 'niedersachsen', 'nordrhein-westfalen', 'rheinland-pfalz', 'saarland', 'sachsen', - 'sachsen-anhalt', 'schleswig-holstein', 'thüringen'] - -france_subregions_geofabrik = ['alsace', 'aquitaine', 'auvergne', 'basse-normandie', 'bourgogne', - 'bretagne', 'centre', 'champagne ardenne', 'corse', 'franche comte', 'guadeloupe', - 'guyane', 'haute-normandie', 'ile-de-france', 'languedoc-roussillon', 'limousin', - 'lorraine', 'martinique', 'mayotte', 'midi-pyrenees', 'nord-pas-de-calais', - 'pays de la loire', 'picardie', 'poitou-charentes', 'provence alpes-cote-d\'azur', - 'reunion', 'rhone-alpes'] -# great-britain_subregions_geofabrik = ['england', 'scotland', 'wales'] - -italy_subregions_geofabrik = [ - 'Centro', 'Isole', 'Nord-Est', 'Nord-Ovest', 'Sud'] - -noregion_geofabrik = ['russia', 'asia'] - geofabrik_regions = ['africa', 'antarctica', 'asia', 'australia-oceania', 'central-america', 'europe', 'north-america', 'south-america'] diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index ad99017f..a25438f3 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -4,7 +4,6 @@ #!/usr/bin/python # import official python packages -import sys import logging import os import struct @@ -161,59 +160,6 @@ def get_region_of_country(county): return region -def get_geofabrik_region_of_country(input_county): - """ - returns the geofabrik region / continent of a given country - the geofabrik region is sometimes written different than the get_region_of_country() region - """ - # search for country match in geofabrik tables to determine region to use for map download - c_translated = translate_country_input_to_geofabrik(input_county) - - region = '' - if c_translated in constants.africa_geofabrik: - region = 'africa' - if c_translated in constants.antarctica_geofabrik: - region = 'antarctica' - if c_translated in constants.asia_geofabrik: - region = 'asia' - if c_translated in constants.australiaoceania_geofabrik: - region = 'australia-oceania' - if c_translated in constants.centralamerica_geofabrik: - region = 'central-america' - if c_translated in constants.europe_geofabrik: - region = 'europe' - if c_translated in constants.northamerica_geofabrik: - region = 'north-america' - if c_translated in constants.northamerica_us_geofabrik: - region = 'north-america/us' - if c_translated in constants.southamerica_geofabrik: - region = 'south-america' - if c_translated in constants.germany_subregions_geofabrik: - region = 'europe\\germany' - if c_translated in constants.noregion_geofabrik: - region = 'no' - if region == '': - log.error('! No Geofabrik region match for country: %s', c_translated) - sys.exit() - - return region - - -def translate_country_input_to_geofabrik(county): - """ - translates the given country to the geofabrik country - the geofabrik country is sometimes written different - """ - # search for user entered country name in translated (to geofabrik). if match continue with matched else continue with user entered country - - try: - c_translated = constants.Translate_Country[f'{county}'] - except KeyError: - c_translated = county - - return c_translated - - def translate_tags_to_keep(name_tags=False, sys_platform='', use_repo=False): """ translates the given tags to format of the operating system. diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index a056ed03..e61183e8 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -15,7 +15,7 @@ # import custom python packages from wahoomc.file_directory_functions import download_url_to_file, unzip -from wahoomc.constants_functions import translate_country_input_to_geofabrik, get_tooling_win_path +from wahoomc.constants_functions import get_tooling_win_path from wahoomc.constants_functions import GeofabrikJson from wahoomc.constants import USER_DL_DIR @@ -160,6 +160,8 @@ def __init__(self, max_days_old, force_download, tiles_from_json=None, border_co self.tiles_from_json = tiles_from_json self.border_countries = border_countries + self.o_geofabrik_json = GeofabrikJson() + self.need_to_dl = [] def should_geofabrik_file_be_downloaded(self): @@ -255,7 +257,8 @@ def check_osm_pbf_file(self): # get translated country (geofabrik) of country # do not download the same file for different countries # --> e.g. China, Hong Kong and Macao, see Issue #11 - transl_c = translate_country_input_to_geofabrik(country) + transl_c = self.o_geofabrik_json.translate_id_no_to_geofabrik( + country) # check for already existing .osm.pbf file map_file_path = glob.glob( @@ -287,16 +290,14 @@ def download_osm_pbf_file(self): """ download countries' OSM files """ - o_geofabrik_json = GeofabrikJson() - for country, item in self.border_countries.items(): try: if item['download'] is True: # build path to downloaded file with translated geofabrik country map_file_path = build_osm_pbf_filepath( - o_geofabrik_json.translate_id_no_to_geofabrik(country)) + self.o_geofabrik_json.translate_id_no_to_geofabrik(country)) # fetch the geofabrik download url to countries' OSM file - url = o_geofabrik_json.get_geofabrik_url(country) + url = self.o_geofabrik_json.get_geofabrik_url(country) download_file(map_file_path, url) self.border_countries[country] = { 'map_file': map_file_path} From 3f3df5445abe5f7a453af2a61d3fd31930e55b78 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 08:02:06 +0100 Subject: [PATCH 16/21] exchange continents and regions with geofabrik content - and adjust/ delete functions + test --- tests/test_constants_geofabrik.py | 11 +++++------ wahoomc/constants.py | 6 ------ wahoomc/constants_functions.py | 12 +++++++++--- wahoomc/geofabrik.py | 3 ++- wahoomc/input.py | 25 ++++++++++++++++++++----- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index b846ed7f..5a0994ef 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -62,6 +62,9 @@ def test_regions_of_geofabrik(self): id_with_no_parent_geofabrik = [] regions_geofabrik = [] + geofabrik_regions = ['africa', 'antarctica', 'asia', 'australia-oceania', + 'central-america', 'europe', 'north-america', 'south-america', 'russia'] + # check against (new) raw representation of geofabrik json for feature in self.o_geofabrik_json.raw_json.features: try: @@ -69,17 +72,13 @@ def test_regions_of_geofabrik(self): except KeyError: id_with_no_parent_geofabrik.append(feature.properties['id']) - geofabrik_regions_w_russia = constants.geofabrik_regions - geofabrik_regions_w_russia.append( - 'russia') - - self.assertCountEqual(geofabrik_regions_w_russia, + self.assertCountEqual(geofabrik_regions, id_with_no_parent_geofabrik) # also check against (new) created dict for region in self.o_geofabrik_json.geofabrik_region_overview: regions_geofabrik.append(region) - self.assertCountEqual(geofabrik_regions_w_russia, + self.assertCountEqual(geofabrik_regions, regions_geofabrik) def test_reading_geofabrik_parent(self): diff --git a/wahoomc/constants.py b/wahoomc/constants.py index 16bf6c66..dd296495 100644 --- a/wahoomc/constants.py +++ b/wahoomc/constants.py @@ -28,9 +28,6 @@ VERSION = '3.2.0' -continents = ['europe', 'unitedstates', 'north-america', 'south-america', 'asia', 'oceania', - 'africa', 'antarctica'] - africa = ['algeria', 'angola', 'benin', 'botswana', 'burkina_faso', 'burundi', 'cameroon', 'canary_islands', 'cape_verde', 'central_african_republic', 'chad', 'comoros', 'cote_d_ivoire', 'democratic_republic_of_the_congo', 'djibouti', 'egypt', @@ -96,9 +93,6 @@ 'south_dakota', 'tennessee', 'texas', 'united_states_virgin_islands', 'utah', 'vermont', 'virginia', 'washington', 'west_virginia', 'wisconsin', 'wyoming'] -geofabrik_regions = ['africa', 'antarctica', 'asia', 'australia-oceania', - 'central-america', 'europe', 'north-america', 'south-america'] - block_download = ['dach', 'alps', 'britain-and-ireland', 'south-africa-and-lesotho', 'us-midwest', 'us-northeast', 'us-pacific', 'us-south', 'us-west'] diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index a25438f3..d2af69bd 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -40,20 +40,25 @@ class GeofabrikJson: raw_json = None geofabrik_overview = {} geofabrik_region_overview = {} + geofabrik_regions = [] def __init__(self): # read geofabrik .json file and fill class-attributes // access is only once if GeofabrikJson.raw_json is None and not GeofabrikJson.geofabrik_overview \ - and not GeofabrikJson.geofabrik_region_overview: - GeofabrikJson.raw_json, GeofabrikJson.geofabrik_overview, GeofabrikJson.geofabrik_region_overview = self.read_geofabrik_json_file() + and not GeofabrikJson.geofabrik_region_overview and not GeofabrikJson.geofabrik_regions: + GeofabrikJson.raw_json, GeofabrikJson.geofabrik_overview, GeofabrikJson.geofabrik_region_overview, GeofabrikJson.geofabrik_regions = self.read_geofabrik_json_file() def read_geofabrik_json_file(self): """ read geofabrik .json file and fill class-attributes + + geofabrik_regions as i defined them are the ones without parent + geofabrik_overview contain all entries, with and without parent """ raw_json = [] geofabrik_overview = {} geofabrik_region_overview = {} + geofabrik_regions = [] with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: raw_json = geojson.load(file_handle) @@ -76,8 +81,9 @@ def read_geofabrik_json_file(self): 'pbf_url': pbf_url} geofabrik_region_overview[id_no] = { 'pbf_url': pbf_url} + geofabrik_regions.append(id_no) - return raw_json, geofabrik_overview, geofabrik_region_overview + return raw_json, geofabrik_overview, geofabrik_region_overview, geofabrik_regions def get_geofabrik_parent_country(self, id_no): """ diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index 995a3acf..181da7a5 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -13,7 +13,7 @@ # import custom python packages from wahoomc.constants import GEOFABRIK_PATH -from wahoomc.constants import special_regions, geofabrik_regions, block_download +from wahoomc.constants import special_regions, block_download from wahoomc.constants_functions import GeofabrikJson log = logging.getLogger('main-logger') @@ -171,6 +171,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): output = [] o_geofabrik_json = GeofabrikJson() + geofabrik_regions = o_geofabrik_json.geofabrik_regions # itterate through tiles and find Geofabrik regions that are in the tiles counter = 1 diff --git a/wahoomc/input.py b/wahoomc/input.py index e698eb4f..e89a751f 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -13,7 +13,6 @@ from tkinter import ttk # import custom python packages -from wahoomc import constants from wahoomc.constants_functions import GeofabrikJson o_geofabrik_json = GeofabrikJson() @@ -155,6 +154,22 @@ def create_checkbox(self, default_value, description, row): return bool_var +def get_countries_of_continent_from_geofabrik(continent): + """ + geofabrik_regions as i defined them are the ones without parent + """ + countries = [] + for region, value in o_geofabrik_json.geofabrik_overview.items(): + try: + if value['parent'] == continent: + countries.append(region) + # regions/ continents do not have a parent + except KeyError: + pass + + return countries + + class InputData(): # pylint: disable=too-many-instance-attributes,too-few-public-methods """ object with all parameters to process maps and default values @@ -322,12 +337,12 @@ def __init__(self, parent, oInputData): # Comboboxes self.cb_continent = ttk.Combobox( - self, values=constants.continents, state="readonly") + self, values=o_geofabrik_json.geofabrik_regions, state="readonly") self.cb_continent.current(0) # pre-select first entry in combobox self.cb_continent.bind("<>", self.callback_continent) self.cb_country = ttk.Combobox( - self, values=constants.europe, state="readonly") + self, state="readonly") # Positioning self.lab_top.grid(column=0, row=0, columnspan=2, padx=5, pady=10) @@ -356,8 +371,8 @@ def callback_continent(self, event): # pylint: disable=unused-argument """ continent = self.cb_continent.get() # get countries for selected region and set for combobox - self.cb_country["values"] = getattr( - constants, continent.replace("-", "")) + self.cb_country["values"] = get_countries_of_continent_from_geofabrik( + continent) self.cb_country.current(0) From c9d0c348efac9226a2a2eec4cb5c1f073e4ada91 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 11:02:15 +0100 Subject: [PATCH 17/21] after running vulture --- wahoomc/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wahoomc/constants.py b/wahoomc/constants.py index dd296495..8edf72c4 100644 --- a/wahoomc/constants.py +++ b/wahoomc/constants.py @@ -24,6 +24,7 @@ WAHOO_MC_DIR = os.path.dirname(__file__) RESOURCES_DIR = os.path.join(WAHOO_MC_DIR, 'resources') TOOLING_WIN_DIR = os.path.join(WAHOO_MC_DIR, 'tooling_win') +# location of repo / python installation - not used ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) VERSION = '3.2.0' From e917f8f251750e3c4bf0ec24032b7afc364d1177 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 11:32:17 +0100 Subject: [PATCH 18/21] changes during coding review - helper function equals the real function - not needed commented out test - autoformatting test --- .vscode/launch.json | 20 +++----------------- tests/test_constants_geofabrik.py | 28 +--------------------------- tests/test_downloader.py | 9 --------- wahoomc/constants_functions.py | 2 +- wahoomc/geofabrik.py | 20 ++++++++++---------- wahoomc/input.py | 2 +- 6 files changed, 16 insertions(+), 65 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 412f06b6..3cc55f5b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -212,7 +212,7 @@ ] }, { - "name": "american_samoa #135", + "name": "solomon_islands geofabrik", "type": "python", "request": "launch", "module": "wahoomc", @@ -220,25 +220,11 @@ "args": [ "cli", "-co", - "american_samoa", - "-c", - "-md", - "100", - ] - }, - { - "name": "commonwealth_of_the_northern_mariana_islands #135", - "type": "python", - "request": "launch", - "module": "wahoomc", - "console": "integratedTerminal", - "args": [ - "cli", - "-co", - "commonwealth_of_the_northern_mariana_islands", + "solomon_islands", "-c", "-md", "100", + "-gt" ] } ] diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index 5a0994ef..ee264a5d 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -47,7 +47,7 @@ def test_if_json_countries_exist_in_geofabrik(self): - some countries are skipped because they do not exist in geofabrik """ for country in self.relevant_countries: - child = self.get_geofabrik_id_by_json_file_country(country) + child = self.o_geofabrik_json.translate_id_no_to_geofabrik(country) self.assertIn(child, {country.replace( '_', '-'), 'us/'+country.replace('_', '-')}) @@ -143,32 +143,6 @@ def test_tranlation_id_to_geofabrik_id(self): 'nebraska') self.assertEqual('us/nebraska', transl_country) - def get_geofabrik_id_by_json_file_country(self, country): - """ - get geofabrik id by country .json filename - """ - # get geofabrik id by country .json filename - # 1. raw - # parent, child = self.o_geofabrik_json.find_geofbrik_parent( - # country) - # if not child: - - # get geofabrik id by country .json filename - # 1. '_' replaced by '-' - child = self.o_geofabrik_json.get_geofabrik_parent_country( - country.replace('_', '-'))[1] - if child: - return child - - # 2. 'us/' prefix and '_' replaced by '-' - child = self.o_geofabrik_json.get_geofabrik_parent_country( - 'us/'+country.replace('_', '-'))[1] - - if child: - return child - - return None - if __name__ == '__main__': unittest.main() diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 44cd8dbf..a04765bf 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -69,15 +69,6 @@ def test_future_timestamp(self): result = older_than_x_days(future, self.max_days_old) self.assertFalse(result) - # def test_download_polygons_file(self): - # """ - # Test the download of land poligons file via URL - # """ - # path = os.path.join(fd_fct.COMMON_DL_DIR, 'land-polygons-split-4326', - # 'land_polygons.shp') - # self.o_downloader.download_file(path, - # 'https://osmdata.openstreetmap.de/download/land-polygons-split-4326.zip', True) - def test_download_geofabrik_file(self): """ Test the download of geofabrik file via URL diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index d2af69bd..1433a177 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -7,7 +7,7 @@ import logging import os import struct -import geojson +import geojson # pylint: disable=import-error # import custom python packages from wahoomc import constants diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index 181da7a5..06e27731 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -2,7 +2,7 @@ functions and object for managing OSM maps """ #!/usr/bin/python -#pylint: skip-file +# pylint: skip-file # import official python packages import sys @@ -54,7 +54,7 @@ def get_tiles_of_country(self): # convert to shape (multipolygon) wanted_region = shape(wanted_map_geom) - #print (f'shape = {wanted_region}') + # print (f'shape = {wanted_region}') # get bounding box (bbox_left, bbox_bottom, bbox_right, bbox_top) = wanted_region.bounds @@ -110,7 +110,7 @@ def get_tiles_of_country(self): log.info('Searching for needed maps, this can take a while.') tiles_of_input = find_needed_countries( bbox_tiles, self.wanted_map, wanted_region) - #print (f'Country= {country}') + # print (f'Country= {country}') return tiles_of_input @@ -154,7 +154,7 @@ def geom(wanted): ident_no = props.get('id', '') if ident_no != wanted: continue - #print (props.get('urls', '')) + # print (props.get('urls', '')) wurls = props.get('urls', '') return (feature.geometry, wurls.get('pbf', '')) return None, None @@ -207,7 +207,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): rgeom = regions.geometry rshape = shape(rgeom) - #print (f'Processing region: {regionname}') + # print (f'Processing region: {regionname}') # check if the region we are processing is needed for the tile we are processing @@ -241,7 +241,7 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): must_download_maps.append(parent) must_download_urls.append( o_geofabrik_json.get_geofabrik_url(parent)) - #parent_added = 1 + # parent_added = 1 else: if regionname not in must_download_maps: must_download_maps.append(regionname) @@ -265,11 +265,11 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): # processing a country and no special sub-region # check if rshape is subset of desired region. If so discard it if wanted_region_polygon.contains(rshape): - #print (f'\t{regionname} is a subset of {wanted_map}, discard it') + # print (f'\t{regionname} is a subset of {wanted_map}, discard it') continue # check if rshape is a superset of desired region. if so discard it if rshape.contains(wanted_region_polygon): - #print (f'\t{regionname} is a superset of {wanted_map}, discard it') + # print (f'\t{regionname} is a superset of {wanted_map}, discard it') # if regionname not in must_download_maps: # must_download_maps.append (regionname) # must_download_urls.append (rurl) @@ -277,13 +277,13 @@ def find_needed_countries(bbox_tiles, wanted_map, wanted_region_polygon): continue # Check if rshape is a part of the tile if rshape.intersects(poly): - #print(f'\tintersecting tile: {regionname} tile={tile}') + # print(f'\tintersecting tile: {regionname} tile={tile}') if regionname not in must_download_maps: must_download_maps.append(regionname) must_download_urls.append(rurl) # If this tile contains the desired region, add it to the output - #print (f'map= {wanted_map}\tmust_download= {must_download_maps}\tparent_added= {parent_added}\tforce_added= {force_added}') + # print (f'map= {wanted_map}\tmust_download= {must_download_maps}\tparent_added= {parent_added}\tforce_added= {force_added}') if wanted_map in must_download_maps or parent_added == 1 or force_added == 1: # first replace any forward slashes with underscores (us/texas to us_texas) must_download_maps = [sub.replace( diff --git a/wahoomc/input.py b/wahoomc/input.py index e89a751f..2201f01e 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -156,7 +156,7 @@ def create_checkbox(self, default_value, description, row): def get_countries_of_continent_from_geofabrik(continent): """ - geofabrik_regions as i defined them are the ones without parent + returns all countries of a continent to be selected in UI """ countries = [] for region, value in o_geofabrik_json.geofabrik_overview.items(): From afb48778ff06dc018a09b79148a0356d1900d3f7 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sat, 18 Feb 2023 11:33:36 +0100 Subject: [PATCH 19/21] move geofabrik json class to own file --- tests/test_constants_geofabrik.py | 4 +- tests/test_downloader.py | 2 +- wahoomc/constants_functions.py | 114 ---------------------------- wahoomc/downloader.py | 2 +- wahoomc/geofabrik.py | 2 +- wahoomc/geofabrik_json.py | 122 ++++++++++++++++++++++++++++++ wahoomc/input.py | 2 +- 7 files changed, 128 insertions(+), 120 deletions(-) create mode 100644 wahoomc/geofabrik_json.py diff --git a/tests/test_constants_geofabrik.py b/tests/test_constants_geofabrik.py index ee264a5d..577f3cd2 100644 --- a/tests/test_constants_geofabrik.py +++ b/tests/test_constants_geofabrik.py @@ -8,8 +8,8 @@ from wahoomc import file_directory_functions as fd_fct from wahoomc.downloader import Downloader from wahoomc import constants -from wahoomc.constants_functions import GeofabrikJson -from wahoomc.constants_functions import CountyIsNoGeofabrikCountry +from wahoomc.geofabrik_json import GeofabrikJson +from wahoomc.geofabrik_json import CountyIsNoGeofabrikCountry # json countries with no geofabrik id partner json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago', diff --git a/tests/test_downloader.py b/tests/test_downloader.py index a04765bf..d42eba75 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -18,7 +18,7 @@ from wahoomc.downloader import Downloader from wahoomc import constants from wahoomc.constants_functions import get_tooling_win_path -from wahoomc.constants_functions import GeofabrikJson +from wahoomc.geofabrik_json import GeofabrikJson class TestDownloader(unittest.TestCase): diff --git a/wahoomc/constants_functions.py b/wahoomc/constants_functions.py index 1433a177..6f509ecf 100644 --- a/wahoomc/constants_functions.py +++ b/wahoomc/constants_functions.py @@ -7,7 +7,6 @@ import logging import os import struct -import geojson # pylint: disable=import-error # import custom python packages from wahoomc import constants @@ -15,7 +14,6 @@ from wahoomc.constants import TOOLING_WIN_DIR from wahoomc.constants import USER_CONFIG_DIR from wahoomc.constants import USER_TOOLING_WIN_DIR -from wahoomc.constants import GEOFABRIK_PATH from wahoomc.file_directory_functions import read_json_file_generic log = logging.getLogger('main-logger') @@ -29,118 +27,6 @@ class TagsToKeepNotFoundError(Exception): """Raised when the specified tags to keep .json file does not exist""" -class CountyIsNoGeofabrikCountry(Exception): - """Raised when actual country is not a geofabrik country""" - - -class GeofabrikJson: - """ - This is a Geofabrik .json processing class for constants in the Geofabrik .json file - """ - raw_json = None - geofabrik_overview = {} - geofabrik_region_overview = {} - geofabrik_regions = [] - - def __init__(self): - # read geofabrik .json file and fill class-attributes // access is only once - if GeofabrikJson.raw_json is None and not GeofabrikJson.geofabrik_overview \ - and not GeofabrikJson.geofabrik_region_overview and not GeofabrikJson.geofabrik_regions: - GeofabrikJson.raw_json, GeofabrikJson.geofabrik_overview, GeofabrikJson.geofabrik_region_overview, GeofabrikJson.geofabrik_regions = self.read_geofabrik_json_file() - - def read_geofabrik_json_file(self): - """ - read geofabrik .json file and fill class-attributes - - geofabrik_regions as i defined them are the ones without parent - geofabrik_overview contain all entries, with and without parent - """ - raw_json = [] - geofabrik_overview = {} - geofabrik_region_overview = {} - geofabrik_regions = [] - - with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: - raw_json = geojson.load(file_handle) - file_handle.close() - - # create a dict with information easy to access because they are often needed - for feature in raw_json.features: - props = feature.properties - id_no = props['id'] - pbf_url = props['urls']['pbf'] - - try: - parent = props['parent'] - - geofabrik_overview[id_no] = { - 'parent': parent, - 'pbf_url': pbf_url} - except KeyError: - geofabrik_overview[id_no] = { - 'pbf_url': pbf_url} - geofabrik_region_overview[id_no] = { - 'pbf_url': pbf_url} - geofabrik_regions.append(id_no) - - return raw_json, geofabrik_overview, geofabrik_region_overview, geofabrik_regions - - def get_geofabrik_parent_country(self, id_no): - """ - Get the parent map/region of a region from the already loaded json data - """ - id_no_translated = self.translate_id_no_to_geofabrik(id_no) - - try: - entry = self.geofabrik_overview[id_no_translated] - if 'parent' in entry: - return (entry['parent'], id_no_translated) - - return ('', id_no_translated) - except KeyError: - return None, None - - def get_geofabrik_url(self, id_no): - """ - Get the map download url from a region with the already loaded json data - """ - id_no_translated = self.translate_id_no_to_geofabrik(id_no) - try: - entry = self.geofabrik_overview[id_no_translated] - if 'pbf_url' in entry: - return entry['pbf_url'] - except KeyError: - pass - - return None - - def is_input_a_geofabrik_id_no(self, id_no): - """ - check if the given input is a geofabrik id number - """ - if id_no in self.geofabrik_overview: - return True - - return False - - def translate_id_no_to_geofabrik(self, country): - """ - get geofabrik id by country .json filename - """ - - if country in self.geofabrik_overview: - return country - - if country.replace('_', '-') in self.geofabrik_overview: - return country.replace('_', '-') - - if 'us/'+country.replace('_', '-') in self.geofabrik_overview: - return 'us/'+country.replace('_', '-') - - # if none of them got triggert --> exception - raise CountyIsNoGeofabrikCountry - - def get_region_of_country(county): """ returns the region / continent of a given country diff --git a/wahoomc/downloader.py b/wahoomc/downloader.py index e61183e8..6f7482c8 100644 --- a/wahoomc/downloader.py +++ b/wahoomc/downloader.py @@ -16,7 +16,7 @@ # import custom python packages from wahoomc.file_directory_functions import download_url_to_file, unzip from wahoomc.constants_functions import get_tooling_win_path -from wahoomc.constants_functions import GeofabrikJson +from wahoomc.geofabrik_json import GeofabrikJson from wahoomc.constants import USER_DL_DIR from wahoomc.constants import USER_MAPS_DIR diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index 06e27731..3725b311 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -14,7 +14,7 @@ # import custom python packages from wahoomc.constants import GEOFABRIK_PATH from wahoomc.constants import special_regions, block_download -from wahoomc.constants_functions import GeofabrikJson +from wahoomc.geofabrik_json import GeofabrikJson log = logging.getLogger('main-logger') diff --git a/wahoomc/geofabrik_json.py b/wahoomc/geofabrik_json.py new file mode 100644 index 00000000..544932ef --- /dev/null +++ b/wahoomc/geofabrik_json.py @@ -0,0 +1,122 @@ +""" +Object for accessing Geofabrik .json file +""" +#!/usr/bin/python + +# import official python packages +import geojson # pylint: disable=import-error + +# import custom python packages +from wahoomc.constants import GEOFABRIK_PATH + + +class CountyIsNoGeofabrikCountry(Exception): + """Raised when actual country is not a geofabrik country""" + + +class GeofabrikJson: + """ + This is a Geofabrik .json processing class for constants in the Geofabrik .json file + """ + raw_json = None + geofabrik_overview = {} + geofabrik_region_overview = {} + geofabrik_regions = [] + + def __init__(self): + # read geofabrik .json file and fill class-attributes // access is only once + if GeofabrikJson.raw_json is None and not GeofabrikJson.geofabrik_overview \ + and not GeofabrikJson.geofabrik_region_overview and not GeofabrikJson.geofabrik_regions: + GeofabrikJson.raw_json, GeofabrikJson.geofabrik_overview, GeofabrikJson.geofabrik_region_overview, GeofabrikJson.geofabrik_regions = self.read_geofabrik_json_file() + + def read_geofabrik_json_file(self): + """ + read geofabrik .json file and fill class-attributes + + geofabrik_regions as i defined them are the ones without parent + geofabrik_overview contain all entries, with and without parent + """ + raw_json = [] + geofabrik_overview = {} + geofabrik_region_overview = {} + geofabrik_regions = [] + + with open(GEOFABRIK_PATH, encoding='utf8') as file_handle: + raw_json = geojson.load(file_handle) + file_handle.close() + + # create a dict with information easy to access because they are often needed + for feature in raw_json.features: + props = feature.properties + id_no = props['id'] + pbf_url = props['urls']['pbf'] + + try: + parent = props['parent'] + + geofabrik_overview[id_no] = { + 'parent': parent, + 'pbf_url': pbf_url} + except KeyError: + geofabrik_overview[id_no] = { + 'pbf_url': pbf_url} + geofabrik_region_overview[id_no] = { + 'pbf_url': pbf_url} + geofabrik_regions.append(id_no) + + return raw_json, geofabrik_overview, geofabrik_region_overview, geofabrik_regions + + def get_geofabrik_parent_country(self, id_no): + """ + Get the parent map/region of a region from the already loaded json data + """ + id_no_translated = self.translate_id_no_to_geofabrik(id_no) + + try: + entry = self.geofabrik_overview[id_no_translated] + if 'parent' in entry: + return (entry['parent'], id_no_translated) + + return ('', id_no_translated) + except KeyError: + return None, None + + def get_geofabrik_url(self, id_no): + """ + Get the map download url from a region with the already loaded json data + """ + id_no_translated = self.translate_id_no_to_geofabrik(id_no) + try: + entry = self.geofabrik_overview[id_no_translated] + if 'pbf_url' in entry: + return entry['pbf_url'] + except KeyError: + pass + + return None + + def is_input_a_geofabrik_id_no(self, id_no): + """ + check if the given input is a geofabrik id number + """ + if id_no in self.geofabrik_overview: + return True + + return False + + def translate_id_no_to_geofabrik(self, country): + """ + get geofabrik id by country .json filename + """ + + if country in self.geofabrik_overview: + return country + + if country.replace('_', '-') in self.geofabrik_overview: + return country.replace('_', '-') + + if 'us/'+country.replace('_', '-') in self.geofabrik_overview: + return 'us/'+country.replace('_', '-') + + # if none of them got triggert --> exception + raise CountyIsNoGeofabrikCountry diff --git a/wahoomc/input.py b/wahoomc/input.py index 2201f01e..4b1e05d7 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -13,7 +13,7 @@ from tkinter import ttk # import custom python packages -from wahoomc.constants_functions import GeofabrikJson +from wahoomc.geofabrik_json import GeofabrikJson o_geofabrik_json = GeofabrikJson() From 4cc009a38c05524239f7393509ccdbb723d495e2 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sun, 19 Feb 2023 04:12:14 +0100 Subject: [PATCH 20/21] Bump to version v4.0.0a0 --- setup.cfg | 2 +- wahoomc/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7b65dc03..74b3ef3f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = wahoomc -version = 3.2.0 +version = 4.0.0a0 author = Benjamin Kreuscher author_email = benni.kreuscher@gmail.com description = Create maps for your Wahoo bike computer based on latest OSM maps diff --git a/wahoomc/constants.py b/wahoomc/constants.py index 8edf72c4..f5bbcf25 100644 --- a/wahoomc/constants.py +++ b/wahoomc/constants.py @@ -26,7 +26,7 @@ TOOLING_WIN_DIR = os.path.join(WAHOO_MC_DIR, 'tooling_win') # location of repo / python installation - not used ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -VERSION = '3.2.0' +VERSION = '4.0.0a0' africa = ['algeria', 'angola', 'benin', 'botswana', 'burkina_faso', 'burundi', From 9c3e7f20572402dc0ca68c504793b6fe623f81d5 Mon Sep 17 00:00:00 2001 From: Benjamin Kreuscher Date: Sun, 19 Feb 2023 04:12:35 +0100 Subject: [PATCH 21/21] delete maps/ content version when last run version < v4.0.0a0 --- wahoomc/setup_functions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wahoomc/setup_functions.py b/wahoomc/setup_functions.py index b25a95a9..bc7f8495 100644 --- a/wahoomc/setup_functions.py +++ b/wahoomc/setup_functions.py @@ -68,13 +68,13 @@ def adjustments_due_to_breaking_changes(): 'Last run was with version %s, deleting files of %s directory due to breaking changes.', version_last_run, USER_OUTPUT_DIR) delete_o5m_pbf_files_in_folder(USER_OUTPUT_DIR) - # version 1.1.0 moved the directories with "processing" files out of the repo folder due to the publishing via PyPI - # old files are moved to the new structure. Can be deleted soon (in version 3.0.x) + # file-names of downloaded .osm.pbf raw mapfiles was adjusted in #182 to focus on geofabrik naming + # other existing files may therefor not be accessed anymore in the future and therefore deleted if version_last_run is None or \ - pkg_resources.parse_version(VERSION) < pkg_resources.parse_version('1.1.0'): + pkg_resources.parse_version(VERSION) < pkg_resources.parse_version('4.0.0a0'): log.info( - 'Last run was with version %s, moving content to new directories due to breaking changes.', version_last_run) - move_old_content_into_new_dirs() + 'Last run was with version %s, deleting files of %s directory due to breaking changes.', version_last_run, USER_MAPS_DIR) + delete_o5m_pbf_files_in_folder(USER_MAPS_DIR) def check_installation_of_required_programs():