diff --git a/aiohttp/connector.py b/aiohttp/connector.py
index cfeded90fb3..6027a38f560 100644
--- a/aiohttp/connector.py
+++ b/aiohttp/connector.py
@@ -199,6 +199,9 @@ def _release(self, key, req, transport, protocol, *, should_close=False):
 
         reader = protocol.reader
         if should_close or (reader.output and not reader.output.at_eof()):
+            conns = self._conns.get(key)
+            if conns is not None and len(conns) == 0:
+                self._conns.pop(key, None)
             transport.close()
         else:
             conns = self._conns.get(key)