diff --git a/docs/FAQ.md b/docs/FAQ.md index 038a77c4..45702a56 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -31,7 +31,7 @@ conda env remove -n gdal-user ``` followed by creating a new environment and install wahoomc into: ``` -conda create -n gdal-user python=3.10 geojson=2.5 gdal=3.6 pip --channel conda-forge --override-channels +conda create -n gdal-user python=3.10 geojson=2.5 gdal=3.4 requests=2.28 shapely=1.8 pip --channel conda-forge --override-channels conda activate gdal-user pip install wahoomc ``` diff --git a/tests/test_osm_maps.py b/tests/test_osm_maps.py index d6d1e888..b14a300d 100644 --- a/tests/test_osm_maps.py +++ b/tests/test_osm_maps.py @@ -6,7 +6,7 @@ import unittest # import custom python packages -from wahoomc.osm_maps_functions import OsmData +from wahoomc.osm_maps_functions import CountryOsmData, XYOsmData from wahoomc.osm_maps_functions import OsmMaps from wahoomc.osm_maps_functions import get_xy_coordinates_from_input # from wahoomc.osm_maps_functions import TileNotFoundError @@ -106,14 +106,16 @@ def process_and_check_border_countries(self, inp_val, calc_border_c, exp_result, """ o_input_data = InputData() + o_input_data.process_border_countries = calc_border_c + if inp_mode == 'country': o_input_data.country = inp_val + o_osm_data = CountryOsmData(o_input_data) elif inp_mode == 'xy_coordinate': o_input_data.xy_coordinates = inp_val - o_input_data.process_border_countries = calc_border_c + o_osm_data = XYOsmData(o_input_data) - o_osm_data = OsmData() - o_osm_data.process_input_of_the_tool(o_input_data) + o_osm_data.process_input_of_the_tool() result = o_osm_data.border_countries @@ -138,8 +140,8 @@ def test_input_country_malta(self): o_input_data = InputData() o_input_data.country = 'malta' - o_osm_data = OsmData() - o_osm_data.process_input_of_the_tool(o_input_data) + o_osm_data = CountryOsmData(o_input_data) + o_osm_data.process_input_of_the_tool() result = o_osm_data.country_name self.assertEqual(result, 'malta') @@ -211,8 +213,10 @@ def test_version_and_tags_of_country_config_file(self): # prevent from downloading land_polygons each time o_input_data.max_days_old = 1000 - o_osm_data = OsmData() - o_downloader = o_osm_data.process_input_of_the_tool(o_input_data) + o_osm_data = CountryOsmData(o_input_data) + o_osm_data.process_input_of_the_tool() + + o_downloader = o_osm_data.get_downloader() # download files marked for download to fill up map_file per country to write to config o_downloader.download_files_if_needed() @@ -224,7 +228,7 @@ def test_version_and_tags_of_country_config_file(self): self.assertTrue( o_osm_maps.tags_are_identical_to_last_run(o_input_data.country)) - country_config = fd_fct.read_json_file(os.path.join( + country_config = fd_fct.read_json_file_country_config(os.path.join( constants.USER_OUTPUT_DIR, o_input_data.country, ".config.json")) self.assertEqual(constants.VERSION, country_config["version_last_run"]) diff --git a/wahoomc/file_directory_functions.py b/wahoomc/file_directory_functions.py index 0d5946a9..5c338dc6 100644 --- a/wahoomc/file_directory_functions.py +++ b/wahoomc/file_directory_functions.py @@ -60,26 +60,24 @@ def create_empty_directories(parent_dir, tiles_from_json, border_countries): os.makedirs(outdir, exist_ok=True) -def read_json_file(json_file_path): +def read_json_file_country_config(json_file_path): """ - read the tiles from the given json file + read the country config (of last run) from the given json file """ log.debug('-' * 80) - log.debug('# Read json file') + log.debug('# Read country config json file') - with open(json_file_path, encoding="utf-8") as json_file: - tiles_from_json = json.load(json_file) - json_file.close() - if tiles_from_json == '': + country_config = read_json_file_generic(json_file_path) + if country_config == '': log.error('! Json file could not be opened.') sys.exit() log.debug( - '+ Use json file %s with %s tiles', json_file.name, len(tiles_from_json)) - log.debug('+ Read json file: OK') + '+ Use country config file %s', json_file_path) + log.debug('+ Read country config json file: OK') - return tiles_from_json + return country_config def read_json_file_generic(json_file_path): diff --git a/wahoomc/geofabrik.py b/wahoomc/geofabrik.py index ea8e9696..8173e1cb 100644 --- a/wahoomc/geofabrik.py +++ b/wahoomc/geofabrik.py @@ -39,6 +39,13 @@ def compose_bouding_box(self, input) -> dict: """calculate bounding box based on geometry or X/Y combination""" pass + def log_tile(self, counter, all): + """ + unified status logging for this class + """ + log.info( + '(+ tile %s of %s) Find needed countries ', counter, all) + class CountryGeofabrik(InformalGeofabrikInterface): """Geofabrik processing for countries""" @@ -82,8 +89,7 @@ def find_needed_countries(self, bbox_tiles, wanted_map, wanted_region_polygon) - for tile in bbox_tiles: # Do progress indicator every 50 tiles if counter % 50 == 0: - log.info( - 'Processing tile %s of %s', counter, len(bbox_tiles)+1) + self.log_tile(counter, len(bbox_tiles)+1) counter += 1 parent_added = 0 @@ -274,8 +280,7 @@ def find_needed_countries(self, bbox_tiles, wanted_map, wanted_region_polygon) - for tile in bbox_tiles: # Do progress indicator every 50 tiles if counter % 50 == 0: - log.info( - 'Processing tile %s of %s', counter, len(bbox_tiles)+1) + self.log_tile(counter, len(bbox_tiles)+1) counter += 1 parent_added = 0 diff --git a/wahoomc/main.py b/wahoomc/main.py index b8acce96..15ac7b38 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -15,7 +15,7 @@ from wahoomc.downloader import download_tooling from wahoomc.osm_maps_functions import OsmMaps -from wahoomc.osm_maps_functions import OsmData +from wahoomc.osm_maps_functions import CountryOsmData, XYOsmData # logging used in the terminal output: # # means top-level command @@ -58,9 +58,14 @@ def run(run_level): if o_input_data.contour: check_installation_of_additional_programs() - o_osm_data = OsmData() + if o_input_data.country: + o_osm_data = CountryOsmData(o_input_data) + elif o_input_data.xy_coordinates: + o_osm_data = XYOsmData(o_input_data) + # Check for not existing or expired files. Mark for download, if dl is needed - o_downloader = o_osm_data.process_input_of_the_tool(o_input_data) + o_osm_data.process_input_of_the_tool() + o_downloader = o_osm_data.get_downloader() # Download files marked for download o_downloader.download_files_if_needed() diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 6c574a1d..153d94b7 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -15,7 +15,7 @@ import logging # import custom python packages -from wahoomc.file_directory_functions import read_json_file, create_empty_directories, write_json_file_generic +from wahoomc.file_directory_functions import read_json_file_country_config, create_empty_directories, write_json_file_generic from wahoomc.constants_functions import translate_tags_to_keep, \ get_tooling_win_path, get_tag_wahoo_xml_path, TagWahooXmlNotFoundError @@ -64,24 +64,31 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""): run given cmd-subprocess and issue error message if wished """ if not cwd: - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: - for line in iter(process.stdout.readline, b''): # b'\n'-separated lines - try: - log.debug('subprocess:%r', line.decode("utf-8").strip()) - except UnicodeDecodeError: - log.debug('subprocess:%r', line.decode("latin-1").strip()) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) as process: - for line in iter(process.stdout.readline, b''): # b'\n'-separated lines - try: - log.debug('subprocess:%r', line.decode("utf-8").strip()) - except UnicodeDecodeError: - log.debug('subprocess:%r', line.decode("latin-1").strip()) + process = subprocess.Popen( # pylint: disable=consider-using-with + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) if error_message and process.wait() != 0: # 0 means success + for line in iter(process.stdout.readline, b''): # b'\n'-separated lines + # print(line.rstrip()) + try: + log.error('subprocess:%r', line.decode("utf-8").strip()) + except UnicodeDecodeError: + log.error('subprocess:%r', line.decode("latin-1").strip()) + log.error(error_message) sys.exit() + else: + for line in iter(process.stdout.readline, b''): # b'\n'-separated lines + # print(line.rstrip()) + try: + log.debug('subprocess:%r', line.decode("utf-8").strip()) + except UnicodeDecodeError: + log.debug('subprocess:%r', line.decode("latin-1").strip()) + def get_timestamp_last_changed(file_path): """ @@ -92,148 +99,206 @@ def get_timestamp_last_changed(file_path): return datetime.fromtimestamp(chg_time).isoformat() -class OsmData(): # pylint: disable=too-few-public-methods +class InformalOsmDataInterface: """ object with all internal parameters to process maps """ - def __init__(self): + def __init__(self, o_input_data): """ - xxx + steps in constructor: + 1. take over input paramters (force_processing is changed in the function further down) + 2. check + download geofabrik file (always) """ self.force_processing = False self.tiles = [] self.border_countries = {} self.country_name = '' - def process_input_of_the_tool(self, o_input_data): + self.o_downloader = Downloader( + o_input_data.max_days_old, o_input_data.force_download, self.border_countries) + # takeover what is given by user first for force_processing + self.force_processing = o_input_data.force_processing + self.process_border_countries = o_input_data.process_border_countries + + log.info('-' * 80) + + # geofabrik file + if self.o_downloader.should_geofabrik_file_be_downloaded(): + self.force_processing = True + self.o_downloader.download_geofabrik_file() + + def process_input_of_the_tool(self): """ Process input: get relevant tiles and if border countries should be calculated The three primary inputs are giving by a separate value each and have separate processing: 1. country name 2. x/y combinations + """ - # Check for not existing or expired files. Mark for download, if dl is needed - # - land polygons file - # - .osm.pbf files + def calc_tiles(self): + """ + calculate relevant tiles for input country or xy coordinate + """ - steps in this function: - 1. take over input paramters (force_processing is changed in the function further down) - 2. check + download geofabrik file if geofabrik-processing - 3. calculate relevant tiles for map creation - 4. calculate border countries for map creation - 5. evaluate the country-name for folder cration during processing - 6. calculate if force_processing should be set to true + def calc_border_countries(self): + """ + calculate the border countries for the given tiles. i.e. + - if CLI/GUI input by user + - if processing x/y coordinates """ + log.info('# Determine involved/border countries') - o_downloader = Downloader( - o_input_data.max_days_old, o_input_data.force_download, self.border_countries) - # takeover what is given by user first for force_processing - self.force_processing = o_input_data.force_processing + # Build list of countries needed + for tile in self.tiles: + for country in tile['countries']: + if country not in self.border_countries: + self.border_countries[country] = {} - log.info('-' * 80) + def log_border_countries(self): + """ + write calculated border countries/involved countries to log + """ + for country in self.border_countries: + log.info('+ Involved country: %s', country) - # geofabrik file - if o_downloader.should_geofabrik_file_be_downloaded(): + # input can be only one country, if there are more than one, + # border countries must be selected + if len(self.border_countries) > 1: + log.info('+ Border countries will be processed') + + def get_downloader(self): + """ + steps in this function: + 1. Check for not existing or expired files. Mark for download, if dl is needed + - land polygons file + - .osm.pbf files + 2. Calculate if force_processing should be set to true + """ + # calc force processing + # Check for not existing or expired files. Mark for download, if dl is needed + self.o_downloader.check_land_polygons_file() + self.o_downloader.check_osm_pbf_file() + + # If one of the files needs to be downloaded, reprocess all files + if self.o_downloader.need_to_dl: self.force_processing = True - o_downloader.download_geofabrik_file() + + return self.o_downloader + + +class CountryOsmData(InformalOsmDataInterface): + """ + object with all internal parameters to process maps for countries + """ + + def __init__(self, o_input_data): + super().__init__(o_input_data) + self.input_country = o_input_data.country + + self.o_geofabrik = CountryGeofabrik(self.input_country) + + def process_input_of_the_tool(self): + """ + steps in this function: + 1. calculate relevant tiles for map creation + 2. calculate border countries for map creation + 3. evaluate the country-name for folder cration during processing + """ # calc tiles - if o_input_data.country: - self.calc_tiles_country(o_input_data) - elif o_input_data.xy_coordinates: - self.calc_tiles_xy(o_input_data) + self.calc_tiles() # calc border countries log.info('-' * 80) - if o_input_data.country: - self.calc_border_countries_country(o_input_data) - elif o_input_data.xy_coordinates: - self.calc_border_countries() + self.calc_border_countries() # log border countries when and when not calculated to output the processed country(s) self.log_border_countries() # calc country name - if o_input_data.country: - # country name is the input argument - self.country_name = o_input_data.country - elif o_input_data.xy_coordinates: - self.calc_country_name_xy() - - # calc force processing - # Check for not existing or expired files. Mark for download, if dl is needed - o_downloader.check_land_polygons_file() - o_downloader.check_osm_pbf_file() - - # If one of the files needs to be downloaded, reprocess all files - if o_downloader.need_to_dl: - self.force_processing = True + self.calc_country_name() - return o_downloader - - def calc_tiles_country(self, o_input_data): + def calc_tiles(self): """ option 1: input a country as parameter, e.g. germany """ - log.info('# Input country: %s.', o_input_data.country) + log.info('# Input country: %s.', self.input_country) # use Geofabrik-URL to calculate the relevant tiles - o_geofabrik = CountryGeofabrik(o_input_data.country) - self.tiles = o_geofabrik.get_tiles_of_wanted_map() - - def calc_tiles_xy(self, o_input_data): - """ - option 2: input a x/y combinations as parameter, e.g. 134/88 or 133/88,130/100 - """ - log.info( - '# Input X/Y coordinates: %s.', o_input_data.xy_coordinates) + self.tiles = self.o_geofabrik.get_tiles_of_wanted_map() - # use Geofabrik-URL to get the relevant tiles - xy_coordinates = get_xy_coordinates_from_input( - o_input_data.xy_coordinates) - - o_geofabrik = XYGeofabrik(xy_coordinates) - # find the tiles for x/y combinations in the geofabrik json files - self.tiles = o_geofabrik.get_tiles_of_wanted_map() - - def calc_border_countries_country(self, o_input_data): + def calc_border_countries(self): """ calculate the border countries for the given tiles when input is a country - if CLI/GUI input by user """ - if o_input_data.process_border_countries: - self.calc_border_countries() + if self.process_border_countries: + super().calc_border_countries() # set the to-be-processed country as border country else: - self.border_countries[o_input_data.country] = {} + self.border_countries[self.input_country] = {} - def calc_border_countries(self): + def calc_country_name(self): """ - calculate the border countries for the given tiles. i.e. - - if CLI/GUI input by user - - if processing x/y coordinates + country name is the country + >1 countries are separated by underscore """ - log.info('# Determine involved/border countries') + self.country_name = self.input_country - # Build list of countries needed - for tile in self.tiles: - for country in tile['countries']: - if country not in self.border_countries: - self.border_countries[country] = {} - def log_border_countries(self): +class XYOsmData(InformalOsmDataInterface): + """ + object with all internal parameters to process maps for XY coordinates + """ + + def __init__(self, o_input_data): + super().__init__(o_input_data) + self.input_xy_coordinates = o_input_data.xy_coordinates + + def process_input_of_the_tool(self): """ - write calculated border countries/involved countries to log + Process input: get relevant tiles and if border countries should be calculated + The three primary inputs are giving by a separate value each and have separate processing: + 1. country name + 2. x/y combinations + + # Check for not existing or expired files. Mark for download, if dl is needed + # - land polygons file + # - .osm.pbf files + + steps in this function: + 1. calculate relevant tiles for map creation + 2. calculate border countries for map creation + 3. evaluate the country-name for folder cration during processing """ - for country in self.border_countries: - log.info('+ Involved country: %s', country) - # input can be only one country, if there are more than one, - # border countries must be selected - if len(self.border_countries) > 1: - log.info('+ Border countries will be processed') + # calc tiles + self.calc_tiles() - def calc_country_name_xy(self): + # calc border countries + log.info('-' * 80) + self.calc_border_countries() + # log border countries when and when not calculated to output the processed country(s) + self.log_border_countries() + + # calc country name + self.calc_country_name() + + def calc_tiles(self): + """ + option 2: input a x/y combinations as parameter, e.g. 134/88 or 133/88,130/100 + """ + log.info('# Input X/Y coordinates: %s.', self.input_xy_coordinates) + + # use Geofabrik-URL to get the relevant tiles + xy_coordinates = get_xy_coordinates_from_input( + self.input_xy_coordinates) + + o_geofabrik = XYGeofabrik(xy_coordinates) + # find the tiles for x/y combinations in the geofabrik json files + self.tiles = o_geofabrik.get_tiles_of_wanted_map() + + def calc_country_name(self): """ country name is the X/Y combinations separated by minus >1 x/y combinations are separated by underscore @@ -395,8 +460,7 @@ def generate_land(self): # create land.dbf, land.prj, land.shp, land.shx if not os.path.isfile(land_file) or self.o_osm_data.force_processing is True: - log.info( - '+ Coordinates: %s,%s. (%s of %s)', tile["x"], tile["y"], tile_count, len(self.o_osm_data.tiles)) + self.log_tile(tile["x"], tile["y"], tile_count) cmd = ['ogr2ogr', '-overwrite', '-skipfailures'] # Try to prevent getting outside of the +/-180 and +/- 90 degrees borders. Normally the +/- 0.1 are there to prevent white lines at border borders. if tile["x"] == 255 or tile["y"] == 255 or tile["x"] == 0 or tile["y"] == 0: @@ -446,8 +510,7 @@ def generate_sea(self): out_file_sea = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'sea.osm') if not os.path.isfile(out_file_sea) or self.o_osm_data.force_processing is True: - log.info( - '+ Coordinates: %s,%s. (%s of %s)', tile["x"], tile["y"], tile_count, len(self.o_osm_data.tiles)) + self.log_tile(tile["x"], tile["y"], tile_count) with open(os.path.join(RESOURCES_DIR, 'sea.osm'), encoding="utf-8") as sea_file: sea_data = sea_file.read() @@ -534,8 +597,7 @@ def split_filtered_country_files_to_tiles(self): for country, val in self.o_osm_data.border_countries.items(): if country not in tile['countries']: continue - log.info( - '+ Coordinates: %s,%s / %s (%s of %s)', tile["x"], tile["y"], country, tile_count, len(self.o_osm_data.tiles)) + self.log_tile(tile["x"], tile["y"], tile_count, country) out_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', f'split-{country}.osm.pbf') out_file_names = os.path.join(USER_OUTPUT_DIR, @@ -605,8 +667,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries): log.info('# Merge splitted tiles with land, elevation and sea') tile_count = 1 for tile in self.o_osm_data.tiles: # pylint: disable=too-many-nested-blocks - log.info( - '+ Coordinates: %s,%s (%s of %s)', tile["x"], tile["y"], tile_count, len(self.o_osm_data.tiles)) + self.log_tile(tile["x"], tile["y"], tile_count) out_tile_dir = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}') @@ -712,8 +773,7 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml): tile_count = 1 for tile in self.o_osm_data.tiles: - log.info( - '+ Coordinates: %s,%s (%s of %s)', tile["x"], tile["y"], tile_count, len(self.o_osm_data.tiles)) + self.log_tile(tile["x"], tile["y"], tile_count) out_file_map = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map') @@ -878,7 +938,7 @@ def tags_are_identical_to_last_run(self, country): tags_are_identical = True try: - country_config = read_json_file(os.path.join( + country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) if not country_config["tags_last_run"] == translate_tags_to_keep(sys_platform=platform.system()) \ or not country_config["name_tags_last_run"] == translate_tags_to_keep(name_tags=True, sys_platform=platform.system()): @@ -895,7 +955,7 @@ def last_changed_is_identical_to_last_run(self, country): last_changed_is_identical = True try: - country_config = read_json_file(os.path.join( + country_config = read_json_file_country_config(os.path.join( USER_OUTPUT_DIR, country, ".config.json")) if not country_config["changed_ts_map_last_run"] == get_timestamp_last_changed(self.o_osm_data.border_countries[country]['map_file']): last_changed_is_identical = False @@ -903,3 +963,14 @@ def last_changed_is_identical_to_last_run(self, country): last_changed_is_identical = False return last_changed_is_identical + + def log_tile(self, tile_x, tile_y, tile_count, additional_info=''): + """ + unified status logging for this class + """ + if additional_info: + log.info('+ (tile %s of %s) Coordinates: %s,%s / %s', tile_count, len(self.o_osm_data.tiles), tile_x, + tile_y, additional_info) + else: + log.info('+ (tile %s of %s) Coordinates: %s,%s', + tile_count, len(self.o_osm_data.tiles), tile_x, tile_y)