Feature: urllib3 instead of curl (#2134)

* feature(urllib3): add urllib3 client

* test(urllib3): test urllib3 client

* test(urllib3): update http test for urllib3

* test(urllib3): use urllib3 client instead of curl

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* style(urllib3): remove unused imports

* style(urllib3): fix pre-commit errors

* ci(urllib3): remove pycurl dependency

* docs(urllib3): add docs

* style(urllib3): fix failing gh-workflow py3.8

* style(urllib3): add mention of ProxyManager

* style(urllib3): fix pre-commit issues

* style(pycurl): remove curl-related code

* feat(urllib3): add missing request features (header, auth, ssl, proxy, redirects)

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix(urllib3): improve styling

* test(urllib3): add new tests

* fix(urllib3): fix request auth

* fix(aws): validate certificate on request

* style(): add missing exports

* feat(aws): add ssl certificate verification from boto

* feat(urllib): try to use certifi.where() if request.ca_certs are not provided

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* ci(pydocstyle): add missing docstring in public class

* test(urllib3): improve test case

* ci(pydocstyle): fix multi-line docstring summary should start at the first line

* feat(urllib3): remove assert_hostname

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* test(boto): add test for get_cert_path returning .pem file path

* test(urllib3): add test for _get_pool_key_parts method

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Asif Saif Uddin <[email protected]>
Co-authored-by: Tomer Nosrati <[email protected]>

.coveragerc

@@ -9,7 +9,7 @@ omit =
     */python?.?/*
     */site-packages/*
     */pypy/*
-    *kombu/async/http/curl.py
+    *kombu/async/http/urllib3_client.py
     *kombu/five.py
     *kombu/transport/mongodb.py
     *kombu/transport/filesystem.py

.github/workflows/linter.yml

@@ -20,7 +20,7 @@ jobs:
         python-version: ["3.12"]
     steps:
       - name: Install system packages
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+        run: sudo apt-get update && sudo apt-get install libssl-dev
       - name: Check out code from GitHub
         uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/python-package.yml

@@ -39,7 +39,7 @@ jobs:
     steps:
     - name: Install apt packages
       if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu')
-      run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+      run: sudo apt-get update && sudo apt-get install libssl-dev
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: useblacksmith/setup-python@v6
@@ -98,7 +98,7 @@ jobs:
 
       steps:
           -   name: Install apt packages
-              run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+              run: sudo apt-get update && sudo apt-get install libssl-dev
 
           -   uses: actions/checkout@v4
           -   name: Set up Python ${{ matrix.python-version }}

docs/reference/index.rst

@@ -71,7 +71,7 @@ Kombu Asynchronous
     kombu.asynchronous.debug
     kombu.asynchronous.http
     kombu.asynchronous.http.base
-    kombu.asynchronous.http.curl
+    kombu.asynchronous.http.urllib3_client
     kombu.asynchronous.aws
     kombu.asynchronous.aws.connection
     kombu.asynchronous.aws.sqs

docs/reference/kombu.asynchronous.http.urllib3_client.rst

@@ -0,0 +1,11 @@
+============================================================
+ Urllib3 HTTP Client Pool - ``kombu.asynchronous.http.urllib3_client``
+============================================================
+
+.. contents::
+    :local:
+.. currentmodule:: kombu.asynchronous.http.urllib3_client
+
+.. automodule:: kombu.asynchronous.http.urllib3_client
+    :members:
+    :undoc-members:

kombu/asynchronous/aws/connection.py

@@ -7,7 +7,7 @@
 
 from vine import promise, transform
 
-from kombu.asynchronous.aws.ext import AWSRequest, get_response
+from kombu.asynchronous.aws.ext import AWSRequest, get_cert_path, get_response
 from kombu.asynchronous.http import Headers, Request, get_client
 
 
@@ -92,7 +92,8 @@ def getrequest(self):
         headers = Headers(self.headers)
         return self.Request(self.path, method=self.method, headers=headers,
                             body=self.body, connect_timeout=self.timeout,
-                            request_timeout=self.timeout, validate_cert=False)
+                            request_timeout=self.timeout,
+                            validate_cert=True, ca_certs=get_cert_path(True))
 
     def getresponse(self, callback=None):
         request = self.getrequest()

kombu/asynchronous/aws/ext.py

@@ -6,6 +6,7 @@
     import boto3
     from botocore import exceptions
     from botocore.awsrequest import AWSRequest
+    from botocore.httpsession import get_cert_path
     from botocore.response import get_response
 except ImportError:
     boto3 = None
@@ -19,8 +20,9 @@ class BotoCoreError(Exception):
     exceptions.BotoCoreError = BotoCoreError
     AWSRequest = _void()
     get_response = _void()
+    get_cert_path = _void()
 
 
 __all__ = (
-    'exceptions', 'AWSRequest', 'get_response'
+    'exceptions', 'AWSRequest', 'get_response', 'get_cert_path',
 )

kombu/asynchronous/http/__init__.py

@@ -1,24 +1,19 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 from kombu.asynchronous import get_event_loop
-from kombu.asynchronous.http.base import Headers, Request, Response
+from kombu.asynchronous.http.base import BaseClient, Headers, Request, Response
 from kombu.asynchronous.hub import Hub
 
-if TYPE_CHECKING:
-    from kombu.asynchronous.http.curl import CurlClient
-
-__all__ = ('Client', 'Headers', 'Response', 'Request')
+__all__ = ('Client', 'Headers', 'Response', 'Request', 'get_client')
 
 
-def Client(hub: Hub | None = None, **kwargs: int) -> CurlClient:
+def Client(hub: Hub | None = None, **kwargs: int) -> BaseClient:
     """Create new HTTP client."""
-    from .curl import CurlClient
-    return CurlClient(hub, **kwargs)
+    from .urllib3_client import Urllib3Client
+    return Urllib3Client(hub, **kwargs)
 
 
-def get_client(hub: Hub | None = None, **kwargs: int) -> CurlClient:
+def get_client(hub: Hub | None = None, **kwargs: int) -> BaseClient:
     """Get or create HTTP client bound to the current event loop."""
     hub = hub or get_event_loop()
     try:

kombu/asynchronous/http/base.py

@@ -16,7 +16,7 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
-__all__ = ('Headers', 'Response', 'Request')
+__all__ = ('Headers', 'Response', 'Request', 'BaseClient')
 
 PYPY = hasattr(sys, 'pypy_version_info')
 
@@ -236,6 +236,12 @@ def header_parser(keyt=normalize_header):
 
 
 class BaseClient:
+    """Base class for HTTP clients.
+
+    This class provides the basic structure and functionality for HTTP clients.
+    Subclasses should implement specific HTTP client behavior.
+    """
+
     Headers = Headers
     Request = Request
     Response = Response