From c12d21e26970b69d6ef543b28c51721b2384c923 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Mon, 26 Feb 2024 19:52:31 -0800 Subject: [PATCH 1/7] rename settings.memory and settings.timeout --- CHANGELOG.md | 5 +++-- osmnx/_http.py | 2 +- osmnx/_nominatim.py | 4 ++-- osmnx/_overpass.py | 10 +++++----- osmnx/elevation.py | 4 ++-- osmnx/settings.py | 18 +++++++++--------- tests/test_osmnx.py | 8 ++++---- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 274079d8f..0f769292e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123) - add type annotations to all public and private functions throughout package (#1107) -- remove functionality previously deprecated in v1 (#1113 #1122) +- remove all functionality previously deprecated in v1 (#1113 #1122) - drop Python 3.8 support (#1106) - bump minimum required numpy version to 1.21 for typing support (#1133) - improve docstrings throughout package (#1116) @@ -25,7 +25,8 @@ Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123) - fix bug in handling requests ConnectionError when querying Overpass status endpoint (#1113) - fix minor bugs throughout to address inconsistencies revealed by type enforcement (#1107 #1114) - rename truncate.truncate_graph_dist max_dist argument to dist (#1134) -- rename settings module's default_accept_language, default_referer, and default_user_agent (#1129) +- rename settings module's default_accept_language, default_referer, and default_user_agent settings (#1129) +- rename settings module's memory and timeout settings (TBD) - rename osm_xml module to \_osm_xml to make it private, as all its functions are private (#1113) - rename private \_downloader module to \_http (#1114) - remove unnecessary private \_api module (#1114) diff --git a/osmnx/_http.py b/osmnx/_http.py index b2a5193d5..cecdf75e9 100644 --- a/osmnx/_http.py +++ b/osmnx/_http.py @@ -208,7 +208,7 @@ def _resolve_host_via_doh(hostname: str) -> str: err_msg = f"Failed to resolve {hostname!r} IP via DoH, requesting host by name" try: url = settings.doh_url_template.format(hostname=hostname) - response = requests.get(url, timeout=settings.timeout) + response = requests.get(url, timeout=settings.requests_timeout) data = response.json() # if we cannot reach DoH server or resolve host, return hostname itself diff --git a/osmnx/_nominatim.py b/osmnx/_nominatim.py index 925b08fe1..179bdb386 100644 --- a/osmnx/_nominatim.py +++ b/osmnx/_nominatim.py @@ -125,12 +125,12 @@ def _nominatim_request( time.sleep(pause) # transmit the HTTP GET request - msg = f"Get {prepared_url} with timeout={settings.timeout}" + msg = f"Get {prepared_url} with timeout={settings.requests_timeout}" utils.log(msg, level=lg.INFO) response = requests.get( url, params=params, - timeout=settings.timeout, + timeout=settings.requests_timeout, headers=_http._get_http_headers(), **settings.requests_kwargs, ) diff --git a/osmnx/_overpass.py b/osmnx/_overpass.py index 9b8951c2d..cb9b7c5fd 100644 --- a/osmnx/_overpass.py +++ b/osmnx/_overpass.py @@ -154,7 +154,7 @@ def _get_overpass_pause( response = requests.get( url, headers=_http._get_http_headers(), - timeout=settings.timeout, + timeout=settings.requests_timeout, **settings.requests_kwargs, ) status = response.text.split("\n")[4] @@ -211,8 +211,8 @@ def _make_overpass_settings() -> str: The `settings.overpass_settings` string formatted with "timeout" and "maxsize" values. """ - maxsize = "" if settings.memory is None else f"[maxsize:{settings.memory}]" - return settings.overpass_settings.format(timeout=settings.timeout, maxsize=maxsize) + maxsize = "" if settings.overpass_memory is None else f"[maxsize:{settings.overpass_memory}]" + return settings.overpass_settings.format(timeout=settings.requests_timeout, maxsize=maxsize) def _make_overpass_polygon_coord_strs(polygon: Polygon | MultiPolygon) -> list[str]: @@ -435,12 +435,12 @@ def _overpass_request( time.sleep(this_pause) # transmit the HTTP POST request - msg = f"Post {prepared_url} with timeout={settings.timeout}" + msg = f"Post {prepared_url} with timeout={settings.requests_timeout}" utils.log(msg, level=lg.INFO) response = requests.post( url, data=data, - timeout=settings.timeout, + timeout=settings.requests_timeout, headers=_http._get_http_headers(), **settings.requests_kwargs, ) diff --git a/osmnx/elevation.py b/osmnx/elevation.py index cb956c4aa..c82dfcf02 100644 --- a/osmnx/elevation.py +++ b/osmnx/elevation.py @@ -278,11 +278,11 @@ def _elevation_request(url: str, pause: float) -> dict[str, Any]: time.sleep(pause) # transmit the HTTP GET request - msg = f"Get {url} with timeout={settings.timeout}" + msg = f"Get {url} with timeout={settings.requests_timeout}" utils.log(msg, level=lg.INFO) response = requests.get( url, - timeout=settings.timeout, + timeout=settings.requests_timeout, headers=_http._get_http_headers(), **settings.requests_kwargs, ) diff --git a/osmnx/settings.py b/osmnx/settings.py index 2a02ae215..65aab5c30 100644 --- a/osmnx/settings.py +++ b/osmnx/settings.py @@ -13,7 +13,7 @@ Default is `["walk"]`. cache_folder : str | Path Path to folder to save/load HTTP response cache files, if the `use_cache` - setting is `True`. Default is `"./cache"`. + setting is True. Default is `"./cache"`. cache_only_mode : bool If True, download network data from Overpass then raise a `CacheOnlyModeInterrupt` error for user to catch. This prevents graph @@ -76,10 +76,6 @@ Maximum area for any part of the geometry in meters: any polygon bigger than this will get divided up for multiple queries to the API. Default is `2500000000`. -memory : int | None - Overpass server memory allocation size for the query, in bytes. If - None, server will use its default allocation size. Use with caution. - Default is `None`. nominatim_endpoint : str The base API url to use for Nominatim queries. Default is `"https://nominatim.openstreetmap.org/"`. @@ -89,6 +85,10 @@ overpass_endpoint : str The base API url to use for Overpass queries. Default is `"https://overpass-api.de/api"`. +overpass_memory : int | None + Overpass server memory allocation size for the query, in bytes. If + None, server will choose its default allocation size. Use with caution. + Default is `None`. overpass_rate_limit : bool If True, check the Overpass server status endpoint for how long to pause before making request. Necessary if server uses slot management, @@ -106,9 +106,9 @@ a local certificate file. More info on options such as auth, cert, verify, and proxies can be found in the requests package advanced docs. Default is `{}`. -timeout : int +requests_timeout : int The timeout interval in seconds for HTTP requests, and (when applicable) - for API to use while running the query. Default is `180`. + for API server to use while running the query. Default is `180`. use_cache : bool If True, cache HTTP responses locally in `cache_folder` instead of calling API repeatedly for the same request. Default is `True`. @@ -152,14 +152,14 @@ log_name: str = "OSMnx" logs_folder: str | Path = "./logs" max_query_area_size: float = 50 * 1000 * 50 * 1000 -memory: int | None = None nominatim_endpoint: str = "https://nominatim.openstreetmap.org/" nominatim_key: str | None = None overpass_endpoint: str = "https://overpass-api.de/api" +overpass_memory: int | None = None overpass_rate_limit: bool = True overpass_settings: str = "[out:json][timeout:{timeout}]{maxsize}" requests_kwargs: dict[str, Any] = {} -timeout: float = 180 +requests_timeout: float = 180 use_cache: bool = True useful_tags_node: list[str] = ["highway", "ref"] useful_tags_way: list[str] = [ diff --git a/tests/test_osmnx.py b/tests/test_osmnx.py index da50e74cf..8dcf9026b 100644 --- a/tests/test_osmnx.py +++ b/tests/test_osmnx.py @@ -361,14 +361,14 @@ def test_find_nearest() -> None: def test_api_endpoints() -> None: """Test different API endpoints.""" - default_timeout = ox.settings.timeout + default_timeout = ox.settings.requests_timeout default_key = ox.settings.nominatim_key default_nominatim_endpoint = ox.settings.nominatim_endpoint default_overpass_endpoint = ox.settings.overpass_endpoint default_overpass_rate_limit = ox.settings.overpass_rate_limit # test good and bad DNS resolution - ox.settings.timeout = 1 + ox.settings.requests_timeout = 1 ip = ox._http._resolve_host_via_doh("overpass-api.de") ip = ox._http._resolve_host_via_doh("AAAAAAAAAAA") _doh_url_template_default = ox.settings.doh_url_template @@ -386,7 +386,7 @@ def test_api_endpoints() -> None: G = ox.graph_from_place(place1, network_type="all") ox.settings.overpass_rate_limit = default_overpass_rate_limit - ox.settings.timeout = default_timeout + ox.settings.requests_timeout = default_timeout params: OrderedDict[str, int | str] = OrderedDict() params["format"] = "json" @@ -565,7 +565,7 @@ def test_graph_from_functions() -> None: network_type="all", ) - ox.settings.memory = 1073741824 + ox.settings.overpass_memory = 1073741824 G = ox.graph_from_point( location_point, dist=500, From f91aaf06aa4e1cf3c6cca38191de189451832c01 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Mon, 26 Feb 2024 19:53:33 -0800 Subject: [PATCH 2/7] rename a couple tests --- tests/test_osmnx.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_osmnx.py b/tests/test_osmnx.py index 8dcf9026b..68e6154e7 100644 --- a/tests/test_osmnx.py +++ b/tests/test_osmnx.py @@ -337,7 +337,7 @@ def test_plots() -> None: fig, ax = ox.plot_figure_ground(G=G) -def test_find_nearest() -> None: +def test_nearest() -> None: """Test nearest node/edge searching.""" # get graph and x/y coords to search G = ox.graph_from_point(location_point, dist=500, network_type="drive", simplify=False) @@ -359,7 +359,7 @@ def test_find_nearest() -> None: _ = ox.distance.nearest_edges(Gp, X[0], Y[0], return_dist=True) -def test_api_endpoints() -> None: +def test_endpoints() -> None: """Test different API endpoints.""" default_timeout = ox.settings.requests_timeout default_key = ox.settings.nominatim_key @@ -433,7 +433,7 @@ def test_api_endpoints() -> None: ox.settings.overpass_endpoint = default_overpass_endpoint -def test_graph_save_load() -> None: # noqa: PLR0915 +def test_save_load() -> None: # noqa: PLR0915 """Test saving/loading graphs to/from disk.""" G = ox.graph_from_point(location_point, dist=500, network_type="drive") @@ -520,7 +520,7 @@ def test_graph_save_load() -> None: # noqa: PLR0915 G = ox.load_graphml(graphml_str=data, node_dtypes=nd, edge_dtypes=ed) -def test_graph_from_functions() -> None: +def test_graph_from() -> None: """Test downloading graphs from Overpass.""" # test subdividing a large geometry (raises a UserWarning) bbox = ox.utils_geo.bbox_from_point((0, 0), dist=1e5, project_utm=True) From f592322d16bc02688e6be23483fbca59f02bd30e Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Mon, 26 Feb 2024 19:57:19 -0800 Subject: [PATCH 3/7] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f769292e..694b70fd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123) - fix minor bugs throughout to address inconsistencies revealed by type enforcement (#1107 #1114) - rename truncate.truncate_graph_dist max_dist argument to dist (#1134) - rename settings module's default_accept_language, default_referer, and default_user_agent settings (#1129) -- rename settings module's memory and timeout settings (TBD) +- rename settings module's memory and timeout settings (#1136) - rename osm_xml module to \_osm_xml to make it private, as all its functions are private (#1113) - rename private \_downloader module to \_http (#1114) - remove unnecessary private \_api module (#1114) From b47ef96dee7eb66efb30fbd771613dd386a3337c Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Wed, 28 Feb 2024 11:47:49 -0800 Subject: [PATCH 4/7] clarify docstring --- osmnx/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnx/settings.py b/osmnx/settings.py index 65aab5c30..f602463ab 100644 --- a/osmnx/settings.py +++ b/osmnx/settings.py @@ -108,7 +108,7 @@ Default is `{}`. requests_timeout : int The timeout interval in seconds for HTTP requests, and (when applicable) - for API server to use while running the query. Default is `180`. + for Overpass server to use for executing the query. Default is `180`. use_cache : bool If True, cache HTTP responses locally in `cache_folder` instead of calling API repeatedly for the same request. Default is `True`. From 42111c2dbff726126a846d7854e9430b01cf02f1 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Wed, 28 Feb 2024 19:08:12 -0800 Subject: [PATCH 5/7] make optional filepath args able to be passed positionally --- osmnx/io.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/osmnx/io.py b/osmnx/io.py index f00942c1c..6cd000193 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -24,8 +24,8 @@ def save_graph_geopackage( G: nx.MultiDiGraph, - *, filepath: str | Path | None = None, + *, directed: bool = False, encoding: str = "utf-8", ) -> None: @@ -74,8 +74,8 @@ def save_graph_geopackage( def save_graphml( G: nx.MultiDiGraph, - *, filepath: str | Path | None = None, + *, gephi: bool = False, encoding: str = "utf-8", ) -> None: @@ -135,6 +135,7 @@ def save_graphml( def load_graphml( filepath: str | Path | None = None, + *, graphml_str: str | None = None, node_dtypes: dict[str, Any] | None = None, edge_dtypes: dict[str, Any] | None = None, @@ -247,8 +248,8 @@ def load_graphml( def save_graph_xml( G: nx.MultiDiGraph, - *, filepath: str | Path | None = None, + *, way_tag_aggs: dict[str, Any] | None = None, encoding: str = "utf-8", ) -> None: From 4541110f98847e5a68e4a3d8dc90ab620519c73b Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 29 Feb 2024 19:40:56 -0800 Subject: [PATCH 6/7] rename settings.nominatim_endpoint and settings.overpass_endpoint --- osmnx/_nominatim.py | 2 +- osmnx/_overpass.py | 8 ++++---- osmnx/settings.py | 16 ++++++++-------- tests/test_osmnx.py | 14 +++++++------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/osmnx/_nominatim.py b/osmnx/_nominatim.py index 179bdb386..3c12ff2b1 100644 --- a/osmnx/_nominatim.py +++ b/osmnx/_nominatim.py @@ -112,7 +112,7 @@ def _nominatim_request( params["key"] = settings.nominatim_key # prepare Nominatim API URL and see if request already exists in cache - url = settings.nominatim_endpoint.rstrip("/") + "/" + request_type + url = settings.nominatim_url.rstrip("/") + "/" + request_type prepared_url = str(requests.Request("GET", url, params=params).prepare().url) cached_response_json = _http._retrieve_from_cache(prepared_url) if isinstance(cached_response_json, list): diff --git a/osmnx/_overpass.py b/osmnx/_overpass.py index cb9b7c5fd..55d5e3278 100644 --- a/osmnx/_overpass.py +++ b/osmnx/_overpass.py @@ -417,10 +417,10 @@ def _overpass_request( response_json """ # resolve url to same IP even if there is server round-robin redirecting - _http._config_dns(settings.overpass_endpoint) + _http._config_dns(settings.overpass_url) # prepare the Overpass API URL and see if request already exists in cache - url = settings.overpass_endpoint.rstrip("/") + "/interpreter" + url = settings.overpass_url.rstrip("/") + "/interpreter" prepared_url = str(requests.Request("GET", url, params=data).prepare().url) cached_response_json = _http._retrieve_from_cache(prepared_url) if isinstance(cached_response_json, dict): @@ -428,7 +428,7 @@ def _overpass_request( # pause then request this URL if pause is None: - this_pause = _get_overpass_pause(settings.overpass_endpoint) + this_pause = _get_overpass_pause(settings.overpass_url) domain = _http._hostname_from_url(url) msg = f"Pausing {this_pause} second(s) before making HTTP POST request to {domain!r}" utils.log(msg, level=lg.INFO) @@ -447,7 +447,7 @@ def _overpass_request( # handle 429 and 504 errors by pausing then recursively re-trying request if response.status_code in {429, 504}: # pragma: no cover - this_pause = error_pause + _get_overpass_pause(settings.overpass_endpoint) + this_pause = error_pause + _get_overpass_pause(settings.overpass_url) msg = ( f"{domain!r} responded {response.status_code} {response.reason}: " f"we'll retry in {this_pause} secs" diff --git a/osmnx/settings.py b/osmnx/settings.py index f602463ab..460bdc0ce 100644 --- a/osmnx/settings.py +++ b/osmnx/settings.py @@ -76,15 +76,12 @@ Maximum area for any part of the geometry in meters: any polygon bigger than this will get divided up for multiple queries to the API. Default is `2500000000`. -nominatim_endpoint : str - The base API url to use for Nominatim queries. Default is - `"https://nominatim.openstreetmap.org/"`. nominatim_key : str | None Your Nominatim API key, if you are using an API instance that requires one. Default is `None`. -overpass_endpoint : str - The base API url to use for Overpass queries. Default is - `"https://overpass-api.de/api"`. +nominatim_url : str + The base API url to use for Nominatim queries. Default is + `"https://nominatim.openstreetmap.org/"`. overpass_memory : int | None Overpass server memory allocation size for the query, in bytes. If None, server will choose its default allocation size. Use with caution. @@ -100,6 +97,9 @@ {maxsize} values are set dynamically by OSMnx when used. To query, for example, historical OSM data as of a certain date: `'[out:json][timeout:90][date:"2019-10-28T19:20:00Z"]'`. Use with caution. +overpass_url : str + The base API url to use for Overpass queries. Default is + `"https://overpass-api.de/api"`. requests_kwargs : dict[str, Any] Optional keyword args to pass to the requests package when connecting to APIs, for example to configure authentication or provide a path to @@ -152,12 +152,12 @@ log_name: str = "OSMnx" logs_folder: str | Path = "./logs" max_query_area_size: float = 50 * 1000 * 50 * 1000 -nominatim_endpoint: str = "https://nominatim.openstreetmap.org/" nominatim_key: str | None = None -overpass_endpoint: str = "https://overpass-api.de/api" +nominatim_url: str = "https://nominatim.openstreetmap.org/" overpass_memory: int | None = None overpass_rate_limit: bool = True overpass_settings: str = "[out:json][timeout:{timeout}]{maxsize}" +overpass_url: str = "https://overpass-api.de/api" requests_kwargs: dict[str, Any] = {} requests_timeout: float = 180 use_cache: bool = True diff --git a/tests/test_osmnx.py b/tests/test_osmnx.py index 68e6154e7..d17a83476 100644 --- a/tests/test_osmnx.py +++ b/tests/test_osmnx.py @@ -361,10 +361,10 @@ def test_nearest() -> None: def test_endpoints() -> None: """Test different API endpoints.""" - default_timeout = ox.settings.requests_timeout + default_requests_timeout = ox.settings.requests_timeout default_key = ox.settings.nominatim_key - default_nominatim_endpoint = ox.settings.nominatim_endpoint - default_overpass_endpoint = ox.settings.overpass_endpoint + default_nominatim_url = ox.settings.nominatim_url + default_overpass_url = ox.settings.overpass_url default_overpass_rate_limit = ox.settings.overpass_rate_limit # test good and bad DNS resolution @@ -381,12 +381,12 @@ def test_endpoints() -> None: # Test changing the Overpass endpoint. # This should fail because we didn't provide a valid endpoint ox.settings.overpass_rate_limit = False - ox.settings.overpass_endpoint = "http://NOT_A_VALID_ENDPOINT/api/" + ox.settings.overpass_url = "http://NOT_A_VALID_ENDPOINT/api/" with pytest.raises(ConnectionError, match="Max retries exceeded with url"): G = ox.graph_from_place(place1, network_type="all") ox.settings.overpass_rate_limit = default_overpass_rate_limit - ox.settings.requests_timeout = default_timeout + ox.settings.requests_timeout = default_requests_timeout params: OrderedDict[str, int | str] = OrderedDict() params["format"] = "json" @@ -429,8 +429,8 @@ def test_endpoints() -> None: response_json = ox._nominatim._nominatim_request(params=params, request_type="lookup") ox.settings.nominatim_key = default_key - ox.settings.nominatim_endpoint = default_nominatim_endpoint - ox.settings.overpass_endpoint = default_overpass_endpoint + ox.settings.nominatim_url = default_nominatim_url + ox.settings.overpass_url = default_overpass_url def test_save_load() -> None: # noqa: PLR0915 From 7ce19f40c3f6af44aafb9658065d833f9d4a6374 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 29 Feb 2024 20:42:36 -0800 Subject: [PATCH 7/7] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 694b70fd5..f80befba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123) - fix minor bugs throughout to address inconsistencies revealed by type enforcement (#1107 #1114) - rename truncate.truncate_graph_dist max_dist argument to dist (#1134) - rename settings module's default_accept_language, default_referer, and default_user_agent settings (#1129) -- rename settings module's memory and timeout settings (#1136) +- rename settings module's memory, nominatim_endpoint, overpass_endpoint, and timeout settings (#1136) - rename osm_xml module to \_osm_xml to make it private, as all its functions are private (#1113) - rename private \_downloader module to \_http (#1114) - remove unnecessary private \_api module (#1114)