Issue997 default connector limit

.gitignore

@@ -1,3 +1,4 @@
+*.swp
 *.bak
 *.egg
 *.egg-info

aiohttp/connector.py

@@ -33,6 +33,8 @@
     32: sha256,
 }
 
+DEFAULT_CONN_LIMIT = 20
+
 
 class Connection(object):
 
@@ -109,14 +111,15 @@ class BaseConnector(object):
     :param keepalive_timeout: (optional) Keep-alive timeout.
     :param bool force_close: Set to True to force close and do reconnect
         after each request (and between redirects).
+    :param limit: The limit of simultaneous connections to the same endpoint.
     :param loop: Optional event loop.
     """
 
     _closed = True  # prevent AttributeError in __del__ if ctor was failed
     _source_traceback = None
 
     def __init__(self, *, conn_timeout=None, keepalive_timeout=_default,
-                 force_close=False, limit=None,
+                 force_close=False, limit=DEFAULT_CONN_LIMIT,
                  loop=None):
 
         if force_close:
@@ -170,6 +173,12 @@ def __del__(self, _warnings=warnings):
             context['source_traceback'] = self._source_traceback
         self._loop.call_exception_handler(context)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
     @property
     def force_close(self):
         """Ultimately close connection on releasing if True."""
@@ -182,7 +191,8 @@ def limit(self):
         Endpoints are the same if they are have equal
         (host, port, is_ssl) triple.
 
-        If limit is None the connector has no limit (default).
+        If limit is None the connector has no limit.
+        The default limit size is 20.
         """
         return self._limit
 

docs/client.rst

@@ -413,6 +413,13 @@ parameter to *connector*::
 
 The example limits amount of parallel connections to `30`.
 
+The default is `20`.
+
+If you explicitly want not to have limits to the same endpoint,
+pass `None`. For example::
+
+    conn = aiohttp.TCPConnector(limit=None)
+
 
 Resolving using custom nameservers
 ----------------------------------
@@ -421,8 +428,8 @@ In order to specify the nameservers to when resolving the hostnames,
 aiodns is required.
 
     from aiohttp.resolver import AsyncResolver
-    
-    
+
+
     resolver = AsyncResolver(nameservers=["8.8.8.8", "8.8.4.4"])
     conn = aiohttp.TCPConnector(resolver=resolver)
 

docs/client_reference.rst

@@ -690,7 +690,7 @@ BaseConnector
 ^^^^^^^^^^^^^
 
 .. class:: BaseConnector(*, conn_timeout=None, keepalive_timeout=30, \
-                         limit=None, \
+                         limit=20, \
                          force_close=False, loop=None)
 
    Base class for all connectors.
@@ -720,6 +720,13 @@ BaseConnector
       recommend to use explicit loops everywhere.
       (optional)
 
+   .. versionchanged:: 0.23
+
+      ``limit`` changed from unlimited (``None``) to 20.
+      Expect a max of up to 20 connections to the same endpoint,
+      if it is not especified.
+      For limitless connections, pass `None` explicitly.
+
    .. attribute:: closed
 
       Read-only property, ``True`` if connector is closed.

tests/test_connector.py

@@ -16,6 +16,7 @@
 from aiohttp import helpers
 from aiohttp.client import ClientResponse
 from aiohttp.connector import Connection
+from aiohttp.connector import DEFAULT_CONN_LIMIT
 
 
 class TestBaseConnector(unittest.TestCase):
@@ -691,9 +692,13 @@ def test_limit_property(self):
 
     def test_limit_property_default(self):
         conn = aiohttp.BaseConnector(loop=self.loop)
-        self.assertIsNone(conn.limit)
+        self.assertEquals(conn.limit, DEFAULT_CONN_LIMIT)
         conn.close()
 
+    def test_limitless(self):
+        with aiohttp.BaseConnector(loop=self.loop, limit=None) as conn:
+            self.assertIsNone(conn.limit)
+
     def test_force_close_and_explicit_keep_alive(self):
         with self.assertRaises(ValueError):
             aiohttp.BaseConnector(loop=self.loop, keepalive_timeout=30,

tests/test_proxy_connector.py

@@ -6,6 +6,16 @@
 from unittest import mock
 
 
+def _create_mocked_loop():
+    """
+    Single place where to define the loop to be used as mock for the tests
+    Delete the `create_future()` from the mock, so it defaults to a new Future
+    """
+    loop_mock = unittest.mock.Mock()
+    del loop_mock.create_future
+    return loop_mock
+
+
 class TestProxyConnector(unittest.TestCase):
 
     def setUp(self):
@@ -44,7 +54,7 @@ def test_connect(self, ClientRequestMock):
         req = ClientRequest('GET', 'http://www.python.org', loop=self.loop)
         self.assertEqual(req.path, '/')
 
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         connector = aiohttp.ProxyConnector('http://proxy.example.com',
                                            loop=loop_mock)
         self.assertIs(loop_mock, connector._loop)
@@ -102,7 +112,7 @@ def test_auth(self, ClientRequestMock):
         self.assertIn('AUTHORIZATION', proxy_req.headers)
         self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers)
 
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         connector = aiohttp.ProxyConnector(
             'http://proxy.example.com', loop=loop_mock,
             proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'))
@@ -144,7 +154,7 @@ def test_auth_from_url(self, ClientRequestMock):
         self.assertIn('AUTHORIZATION', proxy_req.headers)
         self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers)
 
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         connector = aiohttp.ProxyConnector(
             'http://user:[email protected]', loop=loop_mock)
         connector._resolve_host = resolve_mock = unittest.mock.Mock()
@@ -176,7 +186,7 @@ def test_auth__not_modifying_request(self, ClientRequestMock):
         ClientRequestMock.return_value = proxy_req
         proxy_req_headers = dict(proxy_req.headers)
 
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         connector = aiohttp.ProxyConnector(
             'http://user:[email protected]', loop=loop_mock)
         connector._resolve_host = resolve_mock = unittest.mock.Mock()
@@ -192,7 +202,7 @@ def test_auth__not_modifying_request(self, ClientRequestMock):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_connect(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   loop=loop_mock)
         ClientRequestMock.return_value = proxy_req
@@ -225,7 +235,7 @@ def test_https_connect(self, ClientRequestMock):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_connect_runtime_error(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   loop=loop_mock)
         ClientRequestMock.return_value = proxy_req
@@ -255,7 +265,7 @@ def test_https_connect_runtime_error(self, ClientRequestMock):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_connect_http_proxy_error(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   loop=loop_mock)
         ClientRequestMock.return_value = proxy_req
@@ -286,7 +296,7 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_connect_resp_start_error(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   loop=loop_mock)
         ClientRequestMock.return_value = proxy_req
@@ -315,7 +325,7 @@ def test_request_port(self, ClientRequestMock):
                                   loop=self.loop)
         ClientRequestMock.return_value = proxy_req
 
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         connector = aiohttp.ProxyConnector('http://proxy.example.com',
                                            loop=loop_mock)
 
@@ -344,7 +354,7 @@ def test_proxy_auth_property_default(self):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_connect_pass_ssl_context(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   loop=loop_mock)
         ClientRequestMock.return_value = proxy_req
@@ -383,7 +393,7 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock):
 
     @unittest.mock.patch('aiohttp.connector.ClientRequest')
     def test_https_auth(self, ClientRequestMock):
-        loop_mock = unittest.mock.Mock()
+        loop_mock = _create_mocked_loop()
         proxy_req = ClientRequest('GET', 'http://proxy.example.com',
                                   auth=aiohttp.helpers.BasicAuth('user',
                                                                  'pass'),