[DEV] Use geofabrik .json for countries and regions for internal processing (from former constants.py and wahoomc/resources/json)

.vscode/launch.json

@@ -212,33 +212,19 @@
             ]
         },
         {
-            "name": "american_samoa #135",
+            "name": "solomon_islands geofabrik",
             "type": "python",
             "request": "launch",
             "module": "wahoomc",
             "console": "integratedTerminal",
             "args": [
                 "cli",
                 "-co",
-                "american_samoa",
-                "-c",
-                "-md",
-                "100",
-            ]
-        },
-        {
-            "name": "commonwealth_of_the_northern_mariana_islands #135",
-            "type": "python",
-            "request": "launch",
-            "module": "wahoomc",
-            "console": "integratedTerminal",
-            "args": [
-                "cli",
-                "-co",
-                "commonwealth_of_the_northern_mariana_islands",
+                "solomon_islands",
                 "-c",
                 "-md",
                 "100",
+                "-gt"
             ]
         }
     ]

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = wahoomc
-version = 3.2.0
+version = 4.0.0a0
 author = Benjamin Kreuscher
 author_email = [email protected]
 description = Create maps for your Wahoo bike computer based on latest OSM maps 

tests/test_constants.py

@@ -6,43 +6,11 @@
 import mock
 
 
-from wahoomc.constants_functions import translate_country_input_to_geofabrik, translate_tags_to_keep, \
+from wahoomc.constants_functions import translate_tags_to_keep, \
     get_tag_wahoo_xml_path, TagWahooXmlNotFoundError
 from wahoomc.constants import RESOURCES_DIR
 
 
-class TestTranslateCountries(unittest.TestCase):
-    """
-    tests for translating country-constants between geofabrik and other formats
-    """
-
-    def test_translated_countries_to_china(self):
-        """
-        Test countries which have no own geofabrik country but are included in china
-        """
-
-        expected = 'china'
-
-        transl_c = translate_country_input_to_geofabrik('hong_kong')
-        self.assertEqual(expected, transl_c)
-
-        transl_c = translate_country_input_to_geofabrik('macao')
-        self.assertEqual(expected, transl_c)
-
-        transl_c = translate_country_input_to_geofabrik('paracel_islands')
-        self.assertEqual(expected, transl_c)
-
-    def test_translated_countries_no_mapping(self):
-        """
-        Test countries which have no own geofabrik country but are included in china
-        """
-
-        expected = 'germany'
-
-        transl_c = translate_country_input_to_geofabrik('germany')
-        self.assertEqual(expected, transl_c)
-
-
 tags_universal_simple = {"TAGS_TO_KEEP_UNIVERSAL": {
     'access': '',
     'area': 'yes'

tests/test_constants_geofabrik.py

@@ -0,0 +1,148 @@
+"""
+tests for the constants geofabrik & geofabrik file
+"""
+import os
+import unittest
+
+# import custom python packages
+from wahoomc import file_directory_functions as fd_fct
+from wahoomc.downloader import Downloader
+from wahoomc import constants
+from wahoomc.geofabrik_json import GeofabrikJson
+from wahoomc.geofabrik_json import CountyIsNoGeofabrikCountry
+
+# json countries with no geofabrik id partner
+json_file_countries_without_geofabrik_id = ['clipperton_island', 'saint_pierre_and_miquelon', 'trinidad_and_tobago',
+                                            'curacao', 'bonaire_saint_eustatius_and_saba', 'falkland_islands', 'french_guiana',
+                                            'aruba', 'united_states_minor_outlying_islands', 'french_polynesia',
+                                            'norfolk_island', 'wallis_and_futuna', 'northern_mariana_islands', 'paracel_islands', 'united_arab_emirates', 'kuwait', 'qatar', 'spratly_islands', 'singapore', 'brunei', 'bahrain', 'macao', 'cocos_islands', 'christmas_island', 'palestina', 'malaysia', 'saudi_arabia', 'british_indian_ocean_territory', 'israel', 'oman', 'hong_kong', 'south_georgia_and_the_south_sandwich_islands', 'bouvet_island', 'heard_island_and_mcdonald_islands', 'guam', 'commonwealth_of_the_northern_mariana_islands',
+                                            'american_samoa', 'united_states_virgin_islands', 'svalbard_and_jan_mayen', 'united_kingdom', 'åland', 'gibraltar', 'san_marino', 'vatican_city', 'ireland', 'bosnia_and_herzegovina', 'jersey', 'guernsey', 'montserrat', 'bermuda', 'virgin_islands_u.s.', 'dominica', 'saint-barthélemy', 'barbados', 'grenada', 'saint_vincent_and_the_grenadines', 'anguilla', 'saint-martin', 'cayman_islands', 'sint_maarten', 'haiti', 'saint_lucia', 'british_virgin_islands', 'saint_kitts_and_nevis', 'dominican_republic', 'turks_and_caicos_islands', 'antigua_and_barbuda', 'gambia', 'saint_helena', 'cote_d_ivoire', 'western_sahara', 'comoros', 'republic_of_congo', 'democratic_republic_of_the_congo', 'senegal', 'french_southern_territories']
+
+
+class TestConstantsGeofabrik(unittest.TestCase):
+    """
+    tests for constants in geofabrik context
+    """
+
+    def setUp(self):
+        if not os.path.isfile(constants.GEOFABRIK_PATH):
+            o_downloader = Downloader(24, False)
+            o_downloader.download_geofabrik_file()
+
+        self.o_geofabrik_json = GeofabrikJson()
+
+        self.relevant_countries = []
+
+        # calc relevant countries in constructor. these should be valid for the whole class
+        for folder in fd_fct.get_folders_in_folder(os.path.join(constants.RESOURCES_DIR, 'json')):
+            for file in fd_fct.get_files_in_folder(os.path.join(constants.RESOURCES_DIR, 'json', folder)):
+                country = os.path.splitext(file)[0]
+                if country not in json_file_countries_without_geofabrik_id:
+                    self.relevant_countries.append(country)
+
+    def test_if_json_countries_exist_in_geofabrik(self):
+        """
+        go through all files in the wahoo_mc/resources/json directory
+        - check if each country does also exist in geofabrik
+        - some countries are skipped because they do not exist in geofabrik
+        """
+        for country in self.relevant_countries:
+            child = self.o_geofabrik_json.translate_id_no_to_geofabrik(country)
+
+            self.assertIn(child, {country.replace(
+                '_', '-'), 'us/'+country.replace('_', '-')})
+
+    def test_regions_of_geofabrik(self):
+        """
+        go through all files in the wahoo_mc/resources/json directory
+        - check if each country does also exist in geofabrik
+        - some countries are skipped because they do not exist in geofabrik
+        """
+
+        id_with_no_parent_geofabrik = []
+        regions_geofabrik = []
+
+        geofabrik_regions = ['africa', 'antarctica', 'asia', 'australia-oceania',
+                             'central-america', 'europe', 'north-america', 'south-america', 'russia']
+
+        # check against (new) raw representation of geofabrik json
+        for feature in self.o_geofabrik_json.raw_json.features:
+            try:
+                feature.properties['parent']
+            except KeyError:
+                id_with_no_parent_geofabrik.append(feature.properties['id'])
+
+        self.assertCountEqual(geofabrik_regions,
+                              id_with_no_parent_geofabrik)
+
+        # also check against (new) created dict
+        for region in self.o_geofabrik_json.geofabrik_region_overview:
+            regions_geofabrik.append(region)
+        self.assertCountEqual(geofabrik_regions,
+                              regions_geofabrik)
+
+    def test_reading_geofabrik_parent(self):
+        """
+        go through all files in the wahoo_mc/resources/json directory
+        - compare the built URL via existing coding with the url out of the geofabrik json file
+        - some countries are skipped because they do not exist in geofabrik
+        - in addition
+            - skip countries if there is a geofabrik URL (because this URL is different to the built one)
+            - skip countries which do not have a direct geofabrik json entry, mostly they are in another country/region
+        """
+        parent, child = self.o_geofabrik_json.get_geofabrik_parent_country(
+            'germany')
+        self.assertTrue(parent == 'europe' and child == 'germany')
+
+        parent, child = self.o_geofabrik_json.get_geofabrik_parent_country(
+            'baden-wuerttemberg')
+        self.assertTrue(parent == 'germany' and child == 'baden-wuerttemberg')
+
+        parent, child = self.o_geofabrik_json.get_geofabrik_parent_country(
+            'malta')
+        self.assertTrue(parent == 'europe' and child == 'malta')
+
+        parent, child = self.o_geofabrik_json.get_geofabrik_parent_country(
+            'asia')
+        self.assertTrue(parent == '' and child == 'asia')
+
+        with self.assertRaises(CountyIsNoGeofabrikCountry):
+            parent, child = self.o_geofabrik_json.get_geofabrik_parent_country(
+                'xy')
+
+    def test_if_input_is_geofabrik_id(self):
+        """
+        check if the input is a id of the geofabrik file
+        """
+        self.assertTrue(
+            self.o_geofabrik_json.is_input_a_geofabrik_id_no('germany'))
+
+        self.assertTrue(
+            self.o_geofabrik_json.is_input_a_geofabrik_id_no('baden-wuerttemberg'))
+
+        self.assertTrue(
+            self.o_geofabrik_json.is_input_a_geofabrik_id_no('asia'))
+        self.assertFalse(
+            self.o_geofabrik_json.is_input_a_geofabrik_id_no('xy'))
+
+        self.assertFalse(self.o_geofabrik_json.is_input_a_geofabrik_id_no('*'))
+
+    def test_tranlation_id_to_geofabrik_id(self):
+        """
+        get geofabrik id by country .json filename
+        """
+        transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik(
+            'malta')
+        self.assertEqual('malta', transl_country)
+
+        transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik(
+            'solomon_islands')
+        self.assertEqual('solomon-islands', transl_country)
+
+        transl_country = self.o_geofabrik_json.translate_id_no_to_geofabrik(
+            'nebraska')
+        self.assertEqual('us/nebraska', transl_country)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_downloader.py

@@ -13,11 +13,12 @@
 
 from wahoomc.downloader import older_than_x_days
 from wahoomc.downloader import download_file
-from wahoomc.downloader import get_osm_pbf_filepath_url
+from wahoomc.downloader import build_osm_pbf_filepath
 from wahoomc.downloader import download_tooling
 from wahoomc.downloader import Downloader
 from wahoomc import constants
 from wahoomc.constants_functions import get_tooling_win_path
+from wahoomc.geofabrik_json import GeofabrikJson
 
 
 class TestDownloader(unittest.TestCase):
@@ -68,15 +69,6 @@ def test_future_timestamp(self):
         result = older_than_x_days(future, self.max_days_old)
         self.assertFalse(result)
 
-    # def test_download_polygons_file(self):
-    #     """
-    #     Test the download of land poligons file via URL
-    #     """
-    #     path = os.path.join(fd_fct.COMMON_DL_DIR, 'land-polygons-split-4326',
-    #                         'land_polygons.shp')
-    #     self.o_downloader.download_file(path,
-    #                                     'https://osmdata.openstreetmap.de/download/land-polygons-split-4326.zip', True)
-
     def test_download_geofabrik_file(self):
         """
         Test the download of geofabrik file via URL
@@ -106,23 +98,38 @@ def test_download_geofabrik_file_2(self):
         self.assertFalse(
             self.o_downloader.should_geofabrik_file_be_downloaded())
 
-    def test_download_malta_osm_pbf_file(self):
+    def test_building_osm_pbf_filepaths(self):
         """
-        Test the download of geofabrik file via URL
+        Test if composing map file path on the device is correct
+        "/" to "_" is done for "us/" countries
         """
+        self.check_exp_agains_composed_map_file_path(
+            'us/nebraska', 'us_nebraska')
 
-        country = 'malta'
+        self.check_exp_agains_composed_map_file_path(
+            'us/georgia', 'us_georgia')
 
-        path = os.path.join(constants.USER_DL_DIR, 'maps',
-                            f'{country}' + '-latest.osm.pbf')
+        self.check_exp_agains_composed_map_file_path(
+            'malta', 'malta')
 
-        if os.path.exists(path):
-            os.remove(path)
+        self.check_exp_agains_composed_map_file_path(
+            'asia', 'asia')
 
-        map_file_path, url = get_osm_pbf_filepath_url(country)
-        download_file(map_file_path, url, False)
+        self.check_exp_agains_composed_map_file_path(
+            'georgia', 'georgia')
 
-        self.assertTrue(os.path.exists(path))
+    def test_download_osm_pbf_file(self):
+        """
+        Test the download of geofabrik file via URL
+        """
+        self.check_download_osm_pbf_file('malta', os.path.join(constants.USER_DL_DIR, 'maps',
+                                                               'malta' + '-latest.osm.pbf'))
+
+        self.check_download_osm_pbf_file('us/hawaii', os.path.join(constants.USER_DL_DIR, 'maps',
+                                                                   'us_hawaii' + '-latest.osm.pbf'))
+
+        self.check_download_osm_pbf_file('solomon-islands', os.path.join(constants.USER_DL_DIR, 'maps',
+                                                                         'solomon-islands' + '-latest.osm.pbf'))
 
     def test_delete_not_existing_file(self):
         """
@@ -189,6 +196,32 @@ def test_download_macos_files(self):
 
             self.assertTrue(os.path.exists(path))
 
+    def check_exp_agains_composed_map_file_path(self, country, country_map_file_path):
+        """
+        helper function to check a given to-be-path against the composed on based on country
+        """
+        exp_path = os.path.join(constants.USER_DL_DIR, 'maps',
+                                f'{country_map_file_path}' + '-latest.osm.pbf')
+
+        composed_path = build_osm_pbf_filepath(country)
+
+        self.assertEqual(exp_path, composed_path)
+
+    def check_download_osm_pbf_file(self, country, path):
+        """
+        Test if the download of a input country created the wanted map file
+        """
+        o_geofabrik_json = GeofabrikJson()
+
+        if os.path.exists(path):
+            os.remove(path)
+
+        map_file_path = build_osm_pbf_filepath(country)
+        url = o_geofabrik_json.get_geofabrik_url(country)
+        download_file(map_file_path, url, False)
+
+        self.assertTrue(os.path.exists(path))
+
 
 if __name__ == '__main__':
     unittest.main()