-
Notifications
You must be signed in to change notification settings - Fork 184
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Align pool_maxsize for different connection pool implementations. (#535)
* Align pool_maxsize for different connection pool implementations. Signed-off-by: dblock <[email protected]> * Document connection classes and settings. Signed-off-by: dblock <[email protected]> * Undo change in async for backwards compatibility. Signed-off-by: dblock <[email protected]> * Fix: typo. Signed-off-by: dblock <[email protected]> --------- Signed-off-by: dblock <[email protected]>
- Loading branch information
Showing
9 changed files
with
166 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
- [Connection Classes](#connection-classes) | ||
- [Selecting a Connection Class](#selecting-a-connection-class) | ||
- [Urllib3HttpConnection](#urllib3httpconnection) | ||
- [RequestsHttpConnection](#requestshttpconnection) | ||
- [AsyncHttpConnection](#asynchttpconnection) | ||
- [Connection Pooling](#connection-pooling) | ||
|
||
# Connection Classes | ||
|
||
The OpenSearch Python synchronous client supports both the `Urllib3HttpConnection` connection class (default) from the [urllib3](https://pypi.org/project/urllib3/) library, and `RequestsHttpConnection` from the [requests](https://pypi.org/project/requests/) library. We recommend you use the default, unless your application is standardized on `requests`. | ||
|
||
The faster, asynchronous client, implements a class called `AsyncHttpConnection`, which uses [aiohttp](https://pypi.org/project/aiohttp/). | ||
|
||
## Selecting a Connection Class | ||
|
||
### Urllib3HttpConnection | ||
|
||
```python | ||
from opensearchpy import OpenSearch, Urllib3HttpConnection | ||
|
||
client = OpenSearch( | ||
hosts = [{'host': 'localhost', 'port': 9200}], | ||
http_auth = ('admin', 'admin'), | ||
use_ssl = True, | ||
verify_certs = False, | ||
ssl_show_warn = False, | ||
connection_class = Urllib3HttpConnection | ||
) | ||
``` | ||
|
||
### RequestsHttpConnection | ||
|
||
```python | ||
from opensearchpy import OpenSearch, RequestsHttpConnection | ||
|
||
client = OpenSearch( | ||
hosts = [{'host': 'localhost', 'port': 9200}], | ||
http_auth = ('admin', 'admin'), | ||
use_ssl = True, | ||
verify_certs = False, | ||
ssl_show_warn = False, | ||
connection_class = RequestsHttpConnection | ||
) | ||
``` | ||
|
||
### AsyncHttpConnection | ||
|
||
```python | ||
from opensearchpy import AsyncOpenSearch, AsyncHttpConnection | ||
|
||
async def main(): | ||
client = AsyncOpenSearch( | ||
hosts = [{'host': 'localhost', 'port': 9200}], | ||
http_auth = ('admin', 'admin'), | ||
use_ssl = True, | ||
verify_certs = False, | ||
ssl_show_warn = False, | ||
connection_class = AsyncHttpConnection | ||
) | ||
``` | ||
|
||
## Connection Pooling | ||
|
||
The OpenSearch Python client has a connection pool for each `host` value specified during initialization, and a connection pool for HTTP connections to each host implemented in the underlying HTTP libraries. You can adjust the max size of the latter connection pool with `pool_maxsize`. | ||
|
||
If you don't set this value, each connection library implementation will provide its default, which is typically `10`. Changing the pool size may improve performance in some multithreaded scenarios. | ||
|
||
The following example sets the number of connections in the connection pool to 12. | ||
|
||
```python | ||
from opensearchpy import OpenSearch | ||
|
||
client = OpenSearch( | ||
hosts = [{'host': 'localhost', 'port': 9200}], | ||
http_auth = ('admin', 'admin'), | ||
use_ssl = True, | ||
verify_certs = False, | ||
ssl_show_warn = False, | ||
pool_maxsize = 12, | ||
) | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
# | ||
# Modifications Copyright OpenSearch Contributors. See | ||
# GitHub history for details. | ||
|
||
from unittest import TestCase | ||
|
||
from opensearchpy import OpenSearch, RequestsHttpConnection | ||
|
||
|
||
class TestRequests(TestCase): | ||
def test_connection_class(self): | ||
client = OpenSearch(connection_class=RequestsHttpConnection) | ||
self.assertEqual(client.transport.pool_maxsize, None) | ||
self.assertEqual(client.transport.connection_class, RequestsHttpConnection) | ||
self.assertIsInstance( | ||
client.transport.connection_pool.connections[0], RequestsHttpConnection | ||
) | ||
|
||
def test_pool_maxsize(self): | ||
client = OpenSearch(connection_class=RequestsHttpConnection, pool_maxsize=42) | ||
self.assertEqual(client.transport.pool_maxsize, 42) | ||
self.assertEqual( | ||
client.transport.connection_pool.connections[0] | ||
.session.adapters["https://"] | ||
._pool_maxsize, | ||
42, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
# | ||
# Modifications Copyright OpenSearch Contributors. See | ||
# GitHub history for details. | ||
|
||
from unittest import TestCase | ||
|
||
from urllib3.connectionpool import HTTPConnectionPool | ||
|
||
from opensearchpy import OpenSearch, Urllib3HttpConnection | ||
|
||
|
||
class TestUrlLib3(TestCase): | ||
def test_default(self): | ||
client = OpenSearch() | ||
self.assertEqual(client.transport.connection_class, Urllib3HttpConnection) | ||
self.assertEqual(client.transport.pool_maxsize, None) | ||
|
||
def test_connection_class(self): | ||
client = OpenSearch(connection_class=Urllib3HttpConnection) | ||
self.assertEqual(client.transport.connection_class, Urllib3HttpConnection) | ||
self.assertIsInstance( | ||
client.transport.connection_pool.connections[0], Urllib3HttpConnection | ||
) | ||
self.assertIsInstance( | ||
client.transport.connection_pool.connections[0].pool, HTTPConnectionPool | ||
) | ||
|
||
def test_pool_maxsize(self): | ||
client = OpenSearch(connection_class=Urllib3HttpConnection, pool_maxsize=42) | ||
self.assertEqual(client.transport.pool_maxsize, 42) | ||
# https://github.com/python/cpython/blob/3.12/Lib/queue.py#L35 | ||
self.assertEqual( | ||
client.transport.connection_pool.connections[0].pool.pool.maxsize, 42 | ||
) |