From e45d89c204e915412fb6c7104b12e3ae275acf69 Mon Sep 17 00:00:00 2001 From: Benjamin K <53038537+treee111@users.noreply.github.com> Date: Mon, 2 May 2022 20:36:59 +0200 Subject: [PATCH] [FIX] country & regions constants and mapping (#113) * correct contries/region constants and mapping - correct countries per region in constants.py and constants_functions.py - introduce new function to get path of static tile .json file - use new function * unittests for country/region constants and mapping * cherry pick folder functions - from dabe1171ddd543cada6fbf372a8a342cad720716 * launch config for mexico * introducing northamerica_us geofabrik - for processing "mexico -bc" - california etc. have not been able to download * mapping for new_mexico and function to build url (unittesting!) --- .vscode/launch.json | 15 +- common_python/constants.py | 27 +++- common_python/constants_functions.py | 20 ++- common_python/downloader.py | 18 ++- common_python/file_directory_functions.py | 20 +++ common_python/osm_maps_functions.py | 4 +- .../json/south_america/south-america | 30 ---- tests/test_osm_maps.py | 141 ++++++++++++++++-- 8 files changed, 216 insertions(+), 59 deletions(-) delete mode 100644 common_resources/json/south_america/south-america diff --git a/.vscode/launch.json b/.vscode/launch.json index 7e86a94d..90e730d3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -162,6 +162,19 @@ "C:\\VSCode\\python\\wahooMapsCreator\\output\\138\\100\\land.shp", "C:\\VSCode\\python\\wahooMapsCreator\\output\\138\\100\\land1.osm" ] - } + }, + { + "name": "mexico with bc", + "type": "python", + "request": "launch", + "program": "${workspaceRoot}/wahoo_map_creator.py", + "console": "integratedTerminal", + "args": [ + "mexico", + "-md", + "100", + "-bc" + ] + }, ] } \ No newline at end of file diff --git a/common_python/constants.py b/common_python/constants.py index 17569284..1503f20a 100644 --- a/common_python/constants.py +++ b/common_python/constants.py @@ -56,7 +56,8 @@ 'virgin_islands_u.s.': 'central-america', 'western_sahara': 'morocco', 'qatar': 'gcc-states', - 'åland': 'finland' + 'åland': 'finland', + 'new_mexico': 'new-mexico' } continents = ['europe', 'unitedstates', 'north-america', 'south-america', 'asia', 'oceania', @@ -100,7 +101,7 @@ 'bermuda', 'british_virgin_islands', 'canada', 'cayman_islands', 'costa_rica', 'cuba', 'dominica', 'dominican_republic', 'el_salvador', 'greenland', 'grenada', 'guadeloupe', 'guatemala', 'haiti', 'honduras', 'jamaica', 'martinique', - 'mexico', 'montserrat', 'nicaragua', 'panama', 'saint-barth‚lemy', 'saint-martin', + 'mexico', 'montserrat', 'nicaragua', 'panama', 'saint-barthélemy', 'saint-martin', 'saint_kitts_and_nevis', 'saint_lucia', 'saint_vincent_and_the_grenadines', 'sint_maarten', 'turks_and_caicos_islands', 'virgin_islands_u.s.'] @@ -110,11 +111,11 @@ 'pitcairn_islands', 'samoa', 'solomon_islands', 'tokelau', 'tonga', 'tuvalu', 'united_states_minor_outlying_islands', 'vanuatu', 'wallis_and_futuna'] -southamerica = ['australia', 'cook_islands', 'fiji', 'french_polynesia', 'kiribati', - 'marshall_islands', 'micronesia', 'nauru', 'new_caledonia', 'new_zealand', 'niue', - 'norfolk_island', 'northern_mariana_islands', 'palau', 'papua_new_guinea', - 'pitcairn_islands', 'samoa', 'solomon_islands', 'tokelau', 'tonga', 'tuvalu', - 'united_states_minor_outlying_islands', 'vanuatu', 'wallis_and_futuna'] +southamerica = ['argentina', 'aruba', 'bolivia', 'bonaire_saint_eustatius_and_saba', 'brazil', + 'chile', 'clipperton_island', 'colombia', 'curacao', 'ecuador', + 'falkland_islands', 'french_guiana', 'guyana', 'paraguay', 'peru', + 'saint_pierre_and_miquelon', 'suriname', 'trinidad_and_tobago', + 'uruguay', 'venezuela'] unitedstates = ['alabama', 'alaska', 'american_samoa', 'arizona', 'arkansas', 'california', 'colorado', 'commonwealth_of_the_northern_mariana_islands', 'connecticut', @@ -168,6 +169,18 @@ northamerica_geofabrik = ['canada', 'greenland', 'mexico', 'us midwest', 'us northeast', 'us pacific', 'us south', 'us west'] +northamerica_us_geofabrik = ['alabama', 'alaska', 'arizona', 'arkansas', 'california', 'colorado', + 'connecticut', 'delaware', 'district-of-columbia', 'florida', + 'georgia', 'hawaii', 'idaho', 'illinois', 'indiana', 'iowa', 'kansas', + 'kentucky', 'louisiana', 'maine', 'maryland', 'massachusetts', + 'michigan', 'minnesota', 'mississippi', 'missouri', 'montana', + 'nebraska', 'nevada', 'new-hampshire', 'new-jersey', 'new-mexico', + 'new-york', 'north-carolina', 'north-dakota', 'ohio', 'oklahoma' + 'oregon', 'pennsylvania', 'puerto-rico', 'rhode-island', + 'south-carolina', 'south-dakota', 'tennessee', 'texas', + 'us-virgin-islands', 'utah', 'vermont', 'virginia', 'washington', + 'west-virginia', 'wisconsin', 'wyoming'] + southamerica_geofabrik = ['argentina', 'bolivia', 'brazil', 'chile', 'colombia', 'ecuador', 'paraguay', 'peru', 'suriname', 'uruguay', 'venezuela'] diff --git a/common_python/constants_functions.py b/common_python/constants_functions.py index d1f0b013..43567edf 100644 --- a/common_python/constants_functions.py +++ b/common_python/constants_functions.py @@ -5,9 +5,11 @@ # import official python packages import sys +import os # import custom python packages from common_python import constants +from common_python import file_directory_functions as fd_fct def get_region_of_country(county): @@ -24,13 +26,13 @@ def get_region_of_country(county): if county in constants.europe: region = 'europe' if county in constants.northamerica: - region = 'north-america' + region = 'north_america' if county in constants.oceania: region = 'oceania' if county in constants.southamerica: - region = 'south-america' + region = 'south_america' if county in constants.unitedstates: - region = 'united-states' + region = 'united_states' return region @@ -58,6 +60,8 @@ def get_geofabrik_region_of_country(input_county): region = 'europe' if c_translated in constants.northamerica_geofabrik: region = 'north-america' + if c_translated in constants.northamerica_us_geofabrik: + region = 'north-america/us' if c_translated in constants.southamerica_geofabrik: region = 'south-america' if c_translated in constants.germany_subregions_geofabrik: @@ -73,7 +77,7 @@ def get_geofabrik_region_of_country(input_county): def translate_country_input_to_geofabrik(county): """ - translates the given country to the geofabrik country + translates the given country to the geofabrik country the geofabrik country is sometimes written different """ # search for user entered country name in translated (to geofabrik). if match continue with matched else continue with user entered country @@ -84,3 +88,11 @@ def translate_country_input_to_geofabrik(county): c_translated = county return c_translated + + +def get_path_to_static_tile_json(country): + """ + return the path to the static .json file with the files for the given country + """ + return os.path.join(fd_fct.COMMON_DIR, 'json', + get_region_of_country(country), country + '.json') diff --git a/common_python/downloader.py b/common_python/downloader.py index 3c0868ca..dbc8a2cf 100644 --- a/common_python/downloader.py +++ b/common_python/downloader.py @@ -57,6 +57,18 @@ def download_osm_pbf_file(country): """ print(f'+ Trying to download missing map of {country}.') # get .osm.pbf region of country + url = build_url_for_country_osm_pbf_download(country) + # download URL to file + map_file_path = os.path.join( + fd_fct.MAPS_DIR, f'{country}' + '-latest.osm.pbf') + download_file(map_file_path, url, False) + return map_file_path + + +def build_url_for_country_osm_pbf_download(country): + """ + build the geofabrik download url to a countries' OSM file + """ transl_c = const_fct.translate_country_input_to_geofabrik(country) region = const_fct.get_geofabrik_region_of_country(country) if region != 'no': @@ -64,11 +76,7 @@ def download_osm_pbf_file(country): '/' + transl_c + '-latest.osm.pbf' else: url = 'https://download.geofabrik.de/' + transl_c + '-latest.osm.pbf' - # download URL to file - map_file_path = os.path.join( - fd_fct.MAPS_DIR, f'{transl_c}' + '-latest.osm.pbf') - download_file(map_file_path, url, False) - return map_file_path + return url class Downloader: diff --git a/common_python/file_directory_functions.py b/common_python/file_directory_functions.py index 12639d6b..d9010172 100644 --- a/common_python/file_directory_functions.py +++ b/common_python/file_directory_functions.py @@ -6,6 +6,7 @@ # import official python packages import json import os +from os.path import isfile, join import subprocess import sys import zipfile @@ -124,3 +125,22 @@ def write_to_file(file_path, request): with open(file_path, 'wb') as file_handle: for chunk in request.iter_content(chunk_size=1024*100): file_handle.write(chunk) + + +def get_folders_in_folder(folder): + """ + return foldernames of given folder without path as list + """ + onlyfolders = [f for f in os.listdir( + folder) if not isfile(join(folder, f))] + + return onlyfolders + + +def get_files_in_folder(folder): + """ + return filenames of given folder without path as list + """ + onlyfiles = [f for f in os.listdir(folder) if isfile(join(folder, f))] + + return onlyfiles diff --git a/common_python/osm_maps_functions.py b/common_python/osm_maps_functions.py index f751460f..e477d5c5 100644 --- a/common_python/osm_maps_functions.py +++ b/common_python/osm_maps_functions.py @@ -76,8 +76,8 @@ def process_input(self, input_argument, calc_border_countries): # option 2b: use static json files in the repo to calculate relevant tiles else: - json_file_path = os.path.join(fd_fct.COMMON_DIR, 'json', - const_fct.get_region_of_country(input_argument), input_argument + '.json') + json_file_path = const_fct.get_path_to_static_tile_json( + input_argument) self.tiles = fd_fct.read_json_file(json_file_path) # country name is the input argument diff --git a/common_resources/json/south_america/south-america b/common_resources/json/south_america/south-america deleted file mode 100644 index 8e7196d4..00000000 --- a/common_resources/json/south_america/south-america +++ /dev/null @@ -1,30 +0,0 @@ - Volume in drive C is C_Drive - Volume Serial Number is FEB6-5ED7 - - Directory of c:\Windows-Wahoo-Map-Creator-Osmosis\json\south_america - -24-04-2021 18:42 . -24-04-2021 18:42 .. -02-04-2021 03:29 52.718 argentina.json -02-04-2021 03:29 470 aruba.json -02-04-2021 03:29 16.896 bolivia.json -02-04-2021 03:29 655 bonaire_saint_eustatius_and_saba.json -02-04-2021 03:29 95.588 brazil.json -02-04-2021 03:29 27.421 chile.json -02-04-2021 03:29 225 clipperton_island.json -02-04-2021 03:29 17.414 colombia.json -02-04-2021 03:29 542 curacao.json -02-04-2021 03:29 6.568 ecuador.json -02-04-2021 03:29 1.557 falkland_islands.json -02-04-2021 03:29 2.150 french_guiana.json -02-04-2021 03:29 4.631 guyana.json -02-04-2021 03:29 7.801 paraguay.json -02-04-2021 03:29 18.861 peru.json -02-04-2021 03:29 727 saint_pierre_and_miquelon.json -24-04-2021 18:42 0 south-america -02-04-2021 03:29 3.254 suriname.json -02-04-2021 03:29 790 trinidad_and_tobago.json -02-04-2021 03:29 4.820 uruguay.json -02-04-2021 03:29 15.181 venezuela.json - 21 File(s) 278.269 bytes - 2 Dir(s) 695.453.175.808 bytes free diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index 554a6799..4309c484 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -12,6 +12,7 @@ from common_python.input import InputData from common_python import file_directory_functions as fd_fct from common_python import constants_functions as const_fct +from common_python import constants class TestOsmMaps(unittest.TestCase): @@ -112,14 +113,6 @@ def test_calc_without_border_countries(self): self.process_and_check_border_countries( input_file, False, expected_result) - def test_tiles_via_static_json(self): - """ - Test the retrieval of tiles via geofabrik URL against hardcoded data for germany - """ - expected_tiles = [{'x': 138, 'y': 100, 'left': 14.0625, 'top': 36.597889, - 'right': 15.46875, 'bottom': 35.46067, 'countries': ['malta']}] - self.calculate_tiles_via_static_json('malta', expected_tiles) - def process_and_check_border_countries(self, country, calc_border_countries, expected_result): """ helper method to check a country without border countries @@ -130,16 +123,144 @@ def process_and_check_border_countries(self, country, calc_border_countries, exp self.assertEqual(result, expected_result) + +class TestStaticJsons(unittest.TestCase): + """ + tests for static .json files + """ + + def test_tiles_via_static_json(self): + """ + Test the retrieval of tiles via geofabrik URL against hardcoded data for germany + """ + expected_tiles = [{'x': 138, 'y': 100, 'left': 14.0625, 'top': 36.597889, + 'right': 15.46875, 'bottom': 35.46067, 'countries': ['malta']}] + self.calculate_tiles_via_static_json('malta', expected_tiles) + def calculate_tiles_via_static_json(self, country, expected_result): """ use static json files in the repo to calculate relevant tiles """ - json_file_path = os.path.join(fd_fct.COMMON_DIR, 'json', - const_fct.get_region_of_country(country), country + '.json') + + json_file_path = const_fct.get_path_to_static_tile_json(country) tiles = fd_fct.read_json_file(json_file_path) self.assertEqual(tiles, expected_result) + def test_path_of_tile_via_static_json_china(self): + """ + test function to find .json file for china against static path + """ + + country = 'china' + exp_path = os.path.join('asia', country) + + self.calculate_path_to_static_tile_json(country, exp_path) + + def test_path_of_tile_via_static_json_malta(self): + """ + test function to find .json file for malta against static path + """ + + country = 'malta' + exp_path = os.path.join('europe', country) + + self.calculate_path_to_static_tile_json(country, exp_path) + + def test_path_of_tile_via_static_json_mexico(self): + """ + test function to find .json file for mexico against static path + """ + + country = 'mexico' + exp_path = os.path.join('north_america', country) + + self.calculate_path_to_static_tile_json(country, exp_path) + + def test_path_of_tile_via_static_json_bahamas(self): + """ + test function to find .json file for bahamas against static path + """ + + country = 'bahamas' + exp_path = os.path.join('north_america', country) + + self.calculate_path_to_static_tile_json(country, exp_path) + + def test_path_of_tile_via_static_json_chile(self): + """ + test function to find .json file for chile against static path + """ + + country = 'chile' + exp_path = os.path.join('south_america', country) + + self.calculate_path_to_static_tile_json(country, exp_path) + + def calculate_path_to_static_tile_json(self, country, exp_path): + """ + evaluated the .json file with get_path_to_static_tile_json function + - check against the static file path + - check if the file exists + """ + json_file_path = const_fct.get_path_to_static_tile_json(country) + + expected_path = os.path.join(fd_fct.COMMON_DIR, 'json', + exp_path + '.json') + + self.assertEqual(json_file_path, expected_path) + self.assertTrue(os.path.isfile(json_file_path)) + + def test_go_through_folders(self): + """ + go through all files in the common_resources/json directory + - check if correct .json will be evaluated through get_path_to_static_tile_json function + """ + for folder in fd_fct.get_folders_in_folder(os.path.join(fd_fct.COMMON_DIR, 'json')): + for file in fd_fct.get_files_in_folder(os.path.join(fd_fct.COMMON_DIR, 'json', folder)): + country = os.path.splitext(file)[0] + + self.calculate_path_to_static_tile_json( + country, os.path.join(folder, country)) + + def test_go_through_constants(self): + """ + go through the constant for the GUI / country .json to region assignment + - check if correct .json will be evaluated through get_path_to_static_tile_json function + """ + for country in constants.africa: + self.calculate_path_to_static_tile_json( + country, os.path.join("africa", country)) + + for country in constants.antarctica: + self.calculate_path_to_static_tile_json( + country, os.path.join("antarctica", country)) + + for country in constants.asia: + self.calculate_path_to_static_tile_json( + country, os.path.join("asia", country)) + + for country in constants.europe: + self.calculate_path_to_static_tile_json( + country, os.path.join("europe", country)) + + for country in constants.northamerica: + self.calculate_path_to_static_tile_json( + country, os.path.join('north_america', country)) + + for country in constants.oceania: + self.calculate_path_to_static_tile_json( + country, os.path.join("oceania", country)) + + for country in constants.southamerica: + self.calculate_path_to_static_tile_json( + country, os.path.join("south_america", country)) + + for country in constants.unitedstates: + self.calculate_path_to_static_tile_json( + country, os.path.join("united_states", country)) + + if __name__ == '__main__': unittest.main()