Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename settings memory, timeout, nominatim_endpoint, overpass_endpoint #1136

Merged
merged 8 commits into from
Mar 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123)

- add type annotations to all public and private functions throughout package (#1107)
- remove functionality previously deprecated in v1 (#1113 #1122)
- remove all functionality previously deprecated in v1 (#1113 #1122)
- drop Python 3.8 support (#1106)
- bump minimum required numpy version to 1.21 for typing support (#1133)
- improve docstrings throughout package (#1116)
Expand All @@ -25,7 +25,8 @@ Read the v2 [migration guide](https://github.com/gboeing/osmnx/issues/1123)
- fix bug in handling requests ConnectionError when querying Overpass status endpoint (#1113)
- fix minor bugs throughout to address inconsistencies revealed by type enforcement (#1107 #1114)
- rename truncate.truncate_graph_dist max_dist argument to dist (#1134)
- rename settings module's default_accept_language, default_referer, and default_user_agent (#1129)
- rename settings module's default_accept_language, default_referer, and default_user_agent settings (#1129)
- rename settings module's memory, nominatim_endpoint, overpass_endpoint, and timeout settings (#1136)
- rename osm_xml module to \_osm_xml to make it private, as all its functions are private (#1113)
- rename private \_downloader module to \_http (#1114)
- remove unnecessary private \_api module (#1114)
Expand Down
2 changes: 1 addition & 1 deletion osmnx/_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ def _resolve_host_via_doh(hostname: str) -> str:
err_msg = f"Failed to resolve {hostname!r} IP via DoH, requesting host by name"
try:
url = settings.doh_url_template.format(hostname=hostname)
response = requests.get(url, timeout=settings.timeout)
response = requests.get(url, timeout=settings.requests_timeout)
data = response.json()

# if we cannot reach DoH server or resolve host, return hostname itself
Expand Down
6 changes: 3 additions & 3 deletions osmnx/_nominatim.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def _nominatim_request(
params["key"] = settings.nominatim_key

# prepare Nominatim API URL and see if request already exists in cache
url = settings.nominatim_endpoint.rstrip("/") + "/" + request_type
url = settings.nominatim_url.rstrip("/") + "/" + request_type
prepared_url = str(requests.Request("GET", url, params=params).prepare().url)
cached_response_json = _http._retrieve_from_cache(prepared_url)
if isinstance(cached_response_json, list):
Expand All @@ -125,12 +125,12 @@ def _nominatim_request(
time.sleep(pause)

# transmit the HTTP GET request
msg = f"Get {prepared_url} with timeout={settings.timeout}"
msg = f"Get {prepared_url} with timeout={settings.requests_timeout}"
utils.log(msg, level=lg.INFO)
response = requests.get(
url,
params=params,
timeout=settings.timeout,
timeout=settings.requests_timeout,
headers=_http._get_http_headers(),
**settings.requests_kwargs,
)
Expand Down
18 changes: 9 additions & 9 deletions osmnx/_overpass.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def _get_overpass_pause(
response = requests.get(
url,
headers=_http._get_http_headers(),
timeout=settings.timeout,
timeout=settings.requests_timeout,
**settings.requests_kwargs,
)
status = response.text.split("\n")[4]
Expand Down Expand Up @@ -211,8 +211,8 @@ def _make_overpass_settings() -> str:
The `settings.overpass_settings` string formatted with "timeout" and
"maxsize" values.
"""
maxsize = "" if settings.memory is None else f"[maxsize:{settings.memory}]"
return settings.overpass_settings.format(timeout=settings.timeout, maxsize=maxsize)
maxsize = "" if settings.overpass_memory is None else f"[maxsize:{settings.overpass_memory}]"
return settings.overpass_settings.format(timeout=settings.requests_timeout, maxsize=maxsize)


def _make_overpass_polygon_coord_strs(polygon: Polygon | MultiPolygon) -> list[str]:
Expand Down Expand Up @@ -417,37 +417,37 @@ def _overpass_request(
response_json
"""
# resolve url to same IP even if there is server round-robin redirecting
_http._config_dns(settings.overpass_endpoint)
_http._config_dns(settings.overpass_url)

# prepare the Overpass API URL and see if request already exists in cache
url = settings.overpass_endpoint.rstrip("/") + "/interpreter"
url = settings.overpass_url.rstrip("/") + "/interpreter"
prepared_url = str(requests.Request("GET", url, params=data).prepare().url)
cached_response_json = _http._retrieve_from_cache(prepared_url)
if isinstance(cached_response_json, dict):
return cached_response_json

# pause then request this URL
if pause is None:
this_pause = _get_overpass_pause(settings.overpass_endpoint)
this_pause = _get_overpass_pause(settings.overpass_url)
domain = _http._hostname_from_url(url)
msg = f"Pausing {this_pause} second(s) before making HTTP POST request to {domain!r}"
utils.log(msg, level=lg.INFO)
time.sleep(this_pause)

# transmit the HTTP POST request
msg = f"Post {prepared_url} with timeout={settings.timeout}"
msg = f"Post {prepared_url} with timeout={settings.requests_timeout}"
utils.log(msg, level=lg.INFO)
response = requests.post(
url,
data=data,
timeout=settings.timeout,
timeout=settings.requests_timeout,
headers=_http._get_http_headers(),
**settings.requests_kwargs,
)

# handle 429 and 504 errors by pausing then recursively re-trying request
if response.status_code in {429, 504}: # pragma: no cover
this_pause = error_pause + _get_overpass_pause(settings.overpass_endpoint)
this_pause = error_pause + _get_overpass_pause(settings.overpass_url)
msg = (
f"{domain!r} responded {response.status_code} {response.reason}: "
f"we'll retry in {this_pause} secs"
Expand Down
4 changes: 2 additions & 2 deletions osmnx/elevation.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,11 +278,11 @@ def _elevation_request(url: str, pause: float) -> dict[str, Any]:
time.sleep(pause)

# transmit the HTTP GET request
msg = f"Get {url} with timeout={settings.timeout}"
msg = f"Get {url} with timeout={settings.requests_timeout}"
utils.log(msg, level=lg.INFO)
response = requests.get(
url,
timeout=settings.timeout,
timeout=settings.requests_timeout,
headers=_http._get_http_headers(),
**settings.requests_kwargs,
)
Expand Down
7 changes: 4 additions & 3 deletions osmnx/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@

def save_graph_geopackage(
G: nx.MultiDiGraph,
*,
filepath: str | Path | None = None,
*,
directed: bool = False,
encoding: str = "utf-8",
) -> None:
Expand Down Expand Up @@ -74,8 +74,8 @@ def save_graph_geopackage(

def save_graphml(
G: nx.MultiDiGraph,
*,
filepath: str | Path | None = None,
*,
gephi: bool = False,
encoding: str = "utf-8",
) -> None:
Expand Down Expand Up @@ -135,6 +135,7 @@ def save_graphml(

def load_graphml(
filepath: str | Path | None = None,
*,
graphml_str: str | None = None,
node_dtypes: dict[str, Any] | None = None,
edge_dtypes: dict[str, Any] | None = None,
Expand Down Expand Up @@ -247,8 +248,8 @@ def load_graphml(

def save_graph_xml(
G: nx.MultiDiGraph,
*,
filepath: str | Path | None = None,
*,
way_tag_aggs: dict[str, Any] | None = None,
encoding: str = "utf-8",
) -> None:
Expand Down
34 changes: 17 additions & 17 deletions osmnx/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
Default is `["walk"]`.
cache_folder : str | Path
Path to folder to save/load HTTP response cache files, if the `use_cache`
setting is `True`. Default is `"./cache"`.
setting is True. Default is `"./cache"`.
cache_only_mode : bool
If True, download network data from Overpass then raise a
`CacheOnlyModeInterrupt` error for user to catch. This prevents graph
Expand Down Expand Up @@ -76,19 +76,16 @@
Maximum area for any part of the geometry in meters: any polygon bigger
than this will get divided up for multiple queries to the API. Default is
`2500000000`.
memory : int | None
Overpass server memory allocation size for the query, in bytes. If
None, server will use its default allocation size. Use with caution.
Default is `None`.
nominatim_endpoint : str
The base API url to use for Nominatim queries. Default is
`"https://nominatim.openstreetmap.org/"`.
nominatim_key : str | None
Your Nominatim API key, if you are using an API instance that requires
one. Default is `None`.
overpass_endpoint : str
The base API url to use for Overpass queries. Default is
`"https://overpass-api.de/api"`.
nominatim_url : str
The base API url to use for Nominatim queries. Default is
`"https://nominatim.openstreetmap.org/"`.
overpass_memory : int | None
Overpass server memory allocation size for the query, in bytes. If
None, server will choose its default allocation size. Use with caution.
Default is `None`.
overpass_rate_limit : bool
If True, check the Overpass server status endpoint for how long to
pause before making request. Necessary if server uses slot management,
Expand All @@ -100,15 +97,18 @@
{maxsize} values are set dynamically by OSMnx when used.
To query, for example, historical OSM data as of a certain date:
`'[out:json][timeout:90][date:"2019-10-28T19:20:00Z"]'`. Use with caution.
overpass_url : str
The base API url to use for Overpass queries. Default is
`"https://overpass-api.de/api"`.
requests_kwargs : dict[str, Any]
Optional keyword args to pass to the requests package when connecting
to APIs, for example to configure authentication or provide a path to
a local certificate file. More info on options such as auth, cert,
verify, and proxies can be found in the requests package advanced docs.
Default is `{}`.
timeout : int
requests_timeout : int
The timeout interval in seconds for HTTP requests, and (when applicable)
for API to use while running the query. Default is `180`.
for Overpass server to use for executing the query. Default is `180`.
use_cache : bool
If True, cache HTTP responses locally in `cache_folder` instead of calling
API repeatedly for the same request. Default is `True`.
Expand Down Expand Up @@ -152,14 +152,14 @@
log_name: str = "OSMnx"
logs_folder: str | Path = "./logs"
max_query_area_size: float = 50 * 1000 * 50 * 1000
memory: int | None = None
nominatim_endpoint: str = "https://nominatim.openstreetmap.org/"
nominatim_key: str | None = None
overpass_endpoint: str = "https://overpass-api.de/api"
nominatim_url: str = "https://nominatim.openstreetmap.org/"
overpass_memory: int | None = None
overpass_rate_limit: bool = True
overpass_settings: str = "[out:json][timeout:{timeout}]{maxsize}"
overpass_url: str = "https://overpass-api.de/api"
requests_kwargs: dict[str, Any] = {}
timeout: float = 180
requests_timeout: float = 180
use_cache: bool = True
useful_tags_node: list[str] = ["highway", "ref"]
useful_tags_way: list[str] = [
Expand Down
26 changes: 13 additions & 13 deletions tests/test_osmnx.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ def test_plots() -> None:
fig, ax = ox.plot_figure_ground(G=G)


def test_find_nearest() -> None:
def test_nearest() -> None:
"""Test nearest node/edge searching."""
# get graph and x/y coords to search
G = ox.graph_from_point(location_point, dist=500, network_type="drive", simplify=False)
Expand All @@ -359,16 +359,16 @@ def test_find_nearest() -> None:
_ = ox.distance.nearest_edges(Gp, X[0], Y[0], return_dist=True)


def test_api_endpoints() -> None:
def test_endpoints() -> None:
"""Test different API endpoints."""
default_timeout = ox.settings.timeout
default_requests_timeout = ox.settings.requests_timeout
default_key = ox.settings.nominatim_key
default_nominatim_endpoint = ox.settings.nominatim_endpoint
default_overpass_endpoint = ox.settings.overpass_endpoint
default_nominatim_url = ox.settings.nominatim_url
default_overpass_url = ox.settings.overpass_url
default_overpass_rate_limit = ox.settings.overpass_rate_limit

# test good and bad DNS resolution
ox.settings.timeout = 1
ox.settings.requests_timeout = 1
ip = ox._http._resolve_host_via_doh("overpass-api.de")
ip = ox._http._resolve_host_via_doh("AAAAAAAAAAA")
_doh_url_template_default = ox.settings.doh_url_template
Expand All @@ -381,12 +381,12 @@ def test_api_endpoints() -> None:
# Test changing the Overpass endpoint.
# This should fail because we didn't provide a valid endpoint
ox.settings.overpass_rate_limit = False
ox.settings.overpass_endpoint = "http://NOT_A_VALID_ENDPOINT/api/"
ox.settings.overpass_url = "http://NOT_A_VALID_ENDPOINT/api/"
with pytest.raises(ConnectionError, match="Max retries exceeded with url"):
G = ox.graph_from_place(place1, network_type="all")

ox.settings.overpass_rate_limit = default_overpass_rate_limit
ox.settings.timeout = default_timeout
ox.settings.requests_timeout = default_requests_timeout

params: OrderedDict[str, int | str] = OrderedDict()
params["format"] = "json"
Expand Down Expand Up @@ -429,11 +429,11 @@ def test_api_endpoints() -> None:
response_json = ox._nominatim._nominatim_request(params=params, request_type="lookup")

ox.settings.nominatim_key = default_key
ox.settings.nominatim_endpoint = default_nominatim_endpoint
ox.settings.overpass_endpoint = default_overpass_endpoint
ox.settings.nominatim_url = default_nominatim_url
ox.settings.overpass_url = default_overpass_url


def test_graph_save_load() -> None: # noqa: PLR0915
def test_save_load() -> None: # noqa: PLR0915
"""Test saving/loading graphs to/from disk."""
G = ox.graph_from_point(location_point, dist=500, network_type="drive")

Expand Down Expand Up @@ -520,7 +520,7 @@ def test_graph_save_load() -> None: # noqa: PLR0915
G = ox.load_graphml(graphml_str=data, node_dtypes=nd, edge_dtypes=ed)


def test_graph_from_functions() -> None:
def test_graph_from() -> None:
"""Test downloading graphs from Overpass."""
# test subdividing a large geometry (raises a UserWarning)
bbox = ox.utils_geo.bbox_from_point((0, 0), dist=1e5, project_utm=True)
Expand Down Expand Up @@ -565,7 +565,7 @@ def test_graph_from_functions() -> None:
network_type="all",
)

ox.settings.memory = 1073741824
ox.settings.overpass_memory = 1073741824
G = ox.graph_from_point(
location_point,
dist=500,
Expand Down