From 7696a64206f863e6e1fc454b00e1e0b15d16a9b3 Mon Sep 17 00:00:00 2001 From: titusfortner Date: Mon, 11 Dec 2023 15:11:58 -0600 Subject: [PATCH] [py] implement configurable configuration class for the http client --- .../webdriver/chrome/remote_connection.py | 3 + .../webdriver/chromium/remote_connection.py | 10 +- py/selenium/webdriver/common/options.py | 10 ++ .../webdriver/edge/remote_connection.py | 3 + .../webdriver/firefox/remote_connection.py | 17 ++- py/selenium/webdriver/remote/client_config.py | 107 ++++++++++++++++++ .../webdriver/remote/remote_connection.py | 78 +++++++------ py/selenium/webdriver/remote/webdriver.py | 22 +++- .../webdriver/safari/remote_connection.py | 16 ++- 9 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 py/selenium/webdriver/remote/client_config.py diff --git a/py/selenium/webdriver/chrome/remote_connection.py b/py/selenium/webdriver/chrome/remote_connection.py index d20ac581f36d7..b3f278fe5c547 100644 --- a/py/selenium/webdriver/chrome/remote_connection.py +++ b/py/selenium/webdriver/chrome/remote_connection.py @@ -18,6 +18,7 @@ from selenium.webdriver import DesiredCapabilities from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection +from selenium.webdriver.remote.client_config import ClientConfig class ChromeRemoteConnection(ChromiumRemoteConnection): @@ -28,6 +29,7 @@ def __init__( remote_server_addr: str, keep_alive: bool = True, ignore_proxy: typing.Optional[bool] = False, + client_config: ClientConfig = None, ) -> None: super().__init__( remote_server_addr=remote_server_addr, @@ -35,4 +37,5 @@ def __init__( browser_name=ChromeRemoteConnection.browser_name, keep_alive=keep_alive, ignore_proxy=ignore_proxy, + client_config=client_config, ) diff --git a/py/selenium/webdriver/chromium/remote_connection.py b/py/selenium/webdriver/chromium/remote_connection.py index 29d33499111cf..cc14a2b00a9f8 100644 --- a/py/selenium/webdriver/chromium/remote_connection.py +++ b/py/selenium/webdriver/chromium/remote_connection.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +from selenium.webdriver.remote.client_config import ClientConfig from selenium.webdriver.remote.remote_connection import RemoteConnection @@ -26,8 +26,14 @@ def __init__( browser_name: str, keep_alive: bool = True, ignore_proxy: bool = False, + client_config: ClientConfig = None, ) -> None: - super().__init__(remote_server_addr, keep_alive, ignore_proxy) + super().__init__( + remote_server_addr=remote_server_addr, + keep_alive=keep_alive, + ignore_proxy=ignore_proxy, + client_config=client_config, + ) self.browser_name = browser_name commands = self._remote_commands(vendor_prefix) for key, value in commands.items(): diff --git a/py/selenium/webdriver/common/options.py b/py/selenium/webdriver/common/options.py index 5952aa7376095..16ee8e77b8b6e 100644 --- a/py/selenium/webdriver/common/options.py +++ b/py/selenium/webdriver/common/options.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import typing +import warnings from abc import ABCMeta from abc import abstractmethod from enum import Enum @@ -432,6 +433,15 @@ def add_argument(self, argument): def ignore_local_proxy_environment_variables(self) -> None: """By calling this you will ignore HTTP_PROXY and HTTPS_PROXY from being picked up and used.""" + warnings.warn( + "using ignore_local_proxy_environment_variables in Options has been deprecated, " + "instead, create a Proxy instance with ProxyType.DIRECT to ignore proxy settings, " + "pass the proxy instance into a ClientConfig constructor, " + "pass the client config instance into the Webdriver constructor", + DeprecationWarning, + stacklevel=2, + ) + self._ignore_local_proxy = True def to_capabilities(self): diff --git a/py/selenium/webdriver/edge/remote_connection.py b/py/selenium/webdriver/edge/remote_connection.py index 5e4a3739ba0e8..b819e66860031 100644 --- a/py/selenium/webdriver/edge/remote_connection.py +++ b/py/selenium/webdriver/edge/remote_connection.py @@ -18,6 +18,7 @@ from selenium.webdriver import DesiredCapabilities from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection +from selenium.webdriver.remote.client_config import ClientConfig class EdgeRemoteConnection(ChromiumRemoteConnection): @@ -28,6 +29,7 @@ def __init__( remote_server_addr: str, keep_alive: bool = True, ignore_proxy: typing.Optional[bool] = False, + client_config: ClientConfig = None, ) -> None: super().__init__( remote_server_addr=remote_server_addr, @@ -35,4 +37,5 @@ def __init__( browser_name=EdgeRemoteConnection.browser_name, keep_alive=keep_alive, ignore_proxy=ignore_proxy, + client_config=client_config, ) diff --git a/py/selenium/webdriver/firefox/remote_connection.py b/py/selenium/webdriver/firefox/remote_connection.py index 1147a6a6aaadd..6dd51524e5243 100644 --- a/py/selenium/webdriver/firefox/remote_connection.py +++ b/py/selenium/webdriver/firefox/remote_connection.py @@ -14,16 +14,29 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import typing from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.remote.client_config import ClientConfig from selenium.webdriver.remote.remote_connection import RemoteConnection class FirefoxRemoteConnection(RemoteConnection): browser_name = DesiredCapabilities.FIREFOX["browserName"] - def __init__(self, remote_server_addr, keep_alive=True, ignore_proxy=False) -> None: - super().__init__(remote_server_addr, keep_alive, ignore_proxy) + def __init__( + self, + remote_server_addr: str, + keep_alive: bool = True, + ignore_proxy: typing.Optional[bool] = False, + client_config: ClientConfig = None, + ) -> None: + super().__init__( + remote_server_addr=remote_server_addr, + keep_alive=keep_alive, + ignore_proxy=ignore_proxy, + client_config=client_config, + ) self._commands["GET_CONTEXT"] = ("GET", "/session/$sessionId/moz/context") self._commands["SET_CONTEXT"] = ("POST", "/session/$sessionId/moz/context") diff --git a/py/selenium/webdriver/remote/client_config.py b/py/selenium/webdriver/remote/client_config.py new file mode 100644 index 0000000000000..2700b44713236 --- /dev/null +++ b/py/selenium/webdriver/remote/client_config.py @@ -0,0 +1,107 @@ +# Licensed to the Software Freedom Conservancy (SFC) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The SFC licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +from urllib import parse + +from selenium.webdriver.common.proxy import Proxy +from selenium.webdriver.common.proxy import ProxyType + + +class ClientConfig: + def __init__( + self, + remote_server_addr: str, + keep_alive: bool = True, + proxy=None, + ) -> None: + self.remote_server_addr = remote_server_addr + self.keep_alive = keep_alive + self.proxy = proxy + + @property + def remote_server_addr(self) -> str: + return self._remote_server_addr + + @remote_server_addr.setter + def remote_server_addr(self, value: str): + self._remote_server_addr = value + + @property + def keep_alive(self) -> bool: + """:Returns: The keep alive value.""" + return self._keep_alive + + @keep_alive.setter + def keep_alive(self, value: bool) -> None: + """Toggles the keep alive value. + + :Args: + - value: whether to keep the http connection alive + """ + self._keep_alive = value + + @property + def proxy(self) -> Proxy: + """:Returns: The proxy used for communicating to the driver/server.""" + + if self._proxy is None or self._proxy.proxyType == ProxyType.SYSTEM: + self._proxy.sslProxy = os.environ.get("https_proxy", os.environ.get("HTTPS_PROXY")) + self._proxy.httpProxy = os.environ.get("http_proxy", os.environ.get("HTTP_PROXY")) + + return self._proxy + + @proxy.setter + def proxy(self, proxy: Proxy) -> None: + """Provides the information for communicating with the driver or + server. + + :Args: + - value: the proxy information to use to communicate with the driver or server + """ + self._proxy = proxy + + def get_proxy_url(self): + if self.proxy.proxy_type == ProxyType.DIRECT: + return None + elif self.proxy.proxy_type == ProxyType.SYSTEM: + _no_proxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY")) + if _no_proxy: + for npu in _no_proxy.split(","): + npu = npu.strip() + if npu == "*": + return None + n_url = parse.urlparse(npu) + remote_add = parse.urlparse(self.remote_server_addr) + if n_url.netloc: + if remote_add.netloc == n_url.netloc: + return None + else: + if n_url.path in remote_add.netloc: + return None + if self.remote_server_addr.startswith("https://"): + return os.environ.get("https_proxy", os.environ.get("HTTPS_PROXY")) + if self.remote_server_addr.startswith("http://"): + return os.environ.get("http_proxy", os.environ.get("HTTP_PROXY")) + elif self.proxy.proxy_type == ProxyType.MANUAL: + if self.remote_server_addr.startswith("https://"): + return self.proxy.sslProxy + elif self.remote_server_addr.startswith("http://"): + return self.proxy.http_proxy + else: + return None + else: + return None diff --git a/py/selenium/webdriver/remote/remote_connection.py b/py/selenium/webdriver/remote/remote_connection.py index 7cfd9eb4c466c..a5d023fa00e73 100644 --- a/py/selenium/webdriver/remote/remote_connection.py +++ b/py/selenium/webdriver/remote/remote_connection.py @@ -20,6 +20,7 @@ import platform import socket import string +import warnings from base64 import b64encode from urllib import parse @@ -27,10 +28,11 @@ import urllib3 from selenium import __version__ - from . import utils +from .client_config import ClientConfig from .command import Command from .errorhandler import ErrorCode +from .. import Proxy LOGGER = logging.getLogger(__name__) @@ -211,12 +213,6 @@ def get_remote_connection_headers(cls, parsed_url, keep_alive=False): return headers - def _get_proxy_url(self): - if self._url.startswith("https://"): - return os.environ.get("https_proxy", os.environ.get("HTTPS_PROXY")) - if self._url.startswith("http://"): - return os.environ.get("http_proxy", os.environ.get("HTTP_PROXY")) - def _identify_http_proxy_auth(self): url = self._proxy_url url = url[url.find(":") + 3 :] @@ -248,31 +244,42 @@ def _get_connection_manager(self): return urllib3.PoolManager(**pool_manager_init_args) - def __init__(self, remote_server_addr: str, keep_alive: bool = False, ignore_proxy: bool = False): - self.keep_alive = keep_alive - self._url = remote_server_addr - - # Env var NO_PROXY will override this part of the code - _no_proxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY")) - if _no_proxy: - for npu in _no_proxy.split(","): - npu = npu.strip() - if npu == "*": - ignore_proxy = True - break - n_url = parse.urlparse(npu) - remote_add = parse.urlparse(self._url) - if n_url.netloc: - if remote_add.netloc == n_url.netloc: - ignore_proxy = True - break - else: - if n_url.path in remote_add.netloc: - ignore_proxy = True - break - - self._proxy_url = self._get_proxy_url() if not ignore_proxy else None - if keep_alive: + def __init__( + self, + remote_server_addr: str, + keep_alive: bool = True, + ignore_proxy: bool = False, + client_config: ClientConfig = None, + ): + self._client_config = client_config or ClientConfig() + + if remote_server_addr: + warnings.warn( + "setting keep_alive in RemoteConnection() is deprecated, " "set in ClientConfig instance insttead", + DeprecationWarning, + stacklevel=2, + ) + self._client_config.remote_server_addr = remote_server_addr + + if not keep_alive: + warnings.warn( + "setting keep_alive in RemoteConnection() is deprecated, " "set in ClientConfig instance insttead", + DeprecationWarning, + stacklevel=2, + ) + self._client_config.keep_alive = keep_alive + + if ignore_proxy: + warnings.warn( + "setting keep_alive in RemoteConnection() is deprecated, " "set in ClientConfig instance insttead", + DeprecationWarning, + stacklevel=2, + ) + self._client_config.proxy = Proxy(raw={"proxyType": "direct"}) + + self._proxy_url = self._client_config.get_proxy_url() + + if self._client_config.keep_alive: self._conn = self._get_connection_manager() self._commands = remote_commands @@ -296,7 +303,7 @@ def execute(self, command, params): for word in substitute_params: del params[word] data = utils.dump_json(params) - url = f"{self._url}{path}" + url = f"{self._client_config.remote_server_addr}{path}" return self._request(command_info[0], url, body=data) def _request(self, method, url, body=None): @@ -312,12 +319,11 @@ def _request(self, method, url, body=None): """ LOGGER.debug("%s %s %s", method, url, body) parsed_url = parse.urlparse(url) - headers = self.get_remote_connection_headers(parsed_url, self.keep_alive) - response = None + headers = self.get_remote_connection_headers(parsed_url, self._client_config.keep_alive) if body and method not in ("POST", "PUT"): body = None - if self.keep_alive: + if self._client_config.keep_alive: response = self._conn.request(method, url, body=body, headers=headers) statuscode = response.status else: diff --git a/py/selenium/webdriver/remote/webdriver.py b/py/selenium/webdriver/remote/webdriver.py index d6cfcdd29c6d4..4d2e0e8832ab2 100644 --- a/py/selenium/webdriver/remote/webdriver.py +++ b/py/selenium/webdriver/remote/webdriver.py @@ -53,6 +53,7 @@ from selenium.webdriver.support.relative_locator import RelativeBy from .bidi_connection import BidiConnection +from .client_config import ClientConfig from .command import Command from .errorhandler import ErrorHandler from .file_detector import FileDetector @@ -92,7 +93,13 @@ def _create_caps(caps): return {"capabilities": {"firstMatch": [{}], "alwaysMatch": always_match}} -def get_remote_connection(capabilities, command_executor, keep_alive, ignore_local_proxy=False): +def get_remote_connection( + capabilities, + command_executor, + keep_alive, + ignore_local_proxy, + client_config=None, +) -> RemoteConnection: from selenium.webdriver.chrome.remote_connection import ChromeRemoteConnection from selenium.webdriver.edge.remote_connection import EdgeRemoteConnection from selenium.webdriver.firefox.remote_connection import FirefoxRemoteConnection @@ -101,7 +108,12 @@ def get_remote_connection(capabilities, command_executor, keep_alive, ignore_loc candidates = [ChromeRemoteConnection, EdgeRemoteConnection, SafariRemoteConnection, FirefoxRemoteConnection] handler = next((c for c in candidates if c.browser_name == capabilities.get("browserName")), RemoteConnection) - return handler(command_executor, keep_alive=keep_alive, ignore_proxy=ignore_local_proxy) + return handler( + remote_server_addr=command_executor, + keep_alive=keep_alive, + ignore_proxy=ignore_local_proxy, + client_config=client_config, + ) def create_matches(options: List[BaseOptions]) -> Dict: @@ -166,8 +178,9 @@ def __init__( self, command_executor="http://127.0.0.1:4444", keep_alive=True, - file_detector=None, + file_detector: FileDetector = None, options: Union[BaseOptions, List[BaseOptions]] = None, + client_config: ClientConfig = None, ) -> None: """Create a new driver that will issue commands using the wire protocol. @@ -175,7 +188,7 @@ def __init__( :Args: - command_executor - Either a string representing URL of the remote server or a custom remote_connection.RemoteConnection object. Defaults to 'http://127.0.0.1:4444/wd/hub'. - - keep_alive - Whether to configure remote_connection.RemoteConnection to use + - keep_alive - (Deprecated) Whether to configure remote_connection.RemoteConnection to use HTTP keep-alive. Defaults to True. - file_detector - Pass custom file detector object during instantiation. If None, then default LocalFileDetector() will be used. @@ -195,6 +208,7 @@ def __init__( command_executor=command_executor, keep_alive=keep_alive, ignore_local_proxy=_ignore_local_proxy, + client_config=client_config, ) self._is_remote = True self.session_id = None diff --git a/py/selenium/webdriver/safari/remote_connection.py b/py/selenium/webdriver/safari/remote_connection.py index a97f614a98585..bafde257a3baa 100644 --- a/py/selenium/webdriver/safari/remote_connection.py +++ b/py/selenium/webdriver/safari/remote_connection.py @@ -16,14 +16,26 @@ # under the License. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.remote.client_config import ClientConfig from selenium.webdriver.remote.remote_connection import RemoteConnection class SafariRemoteConnection(RemoteConnection): browser_name = DesiredCapabilities.SAFARI["browserName"] - def __init__(self, remote_server_addr: str, keep_alive: bool = True, ignore_proxy: bool = False) -> None: - super().__init__(remote_server_addr, keep_alive, ignore_proxy) + def __init__( + self, + remote_server_addr: str, + keep_alive: bool = True, + ignore_proxy: bool = False, + client_config: ClientConfig = None, + ) -> None: + super().__init__( + remote_server_addr=remote_server_addr, + keep_alive=keep_alive, + ignore_proxy=ignore_proxy, + client_config=client_config, + ) self._commands["GET_PERMISSIONS"] = ("GET", "/session/$sessionId/apple/permissions") self._commands["SET_PERMISSIONS"] = ("POST", "/session/$sessionId/apple/permissions")