cleanup recursive url session (#10863)

langchain-ai · Sep 21, 2023 · 5097007 · 5097007
1 parent 777b33b
commit 5097007
Showing 1 changed file with 10 additions and 4 deletions.
diff --git a/libs/langchain/langchain/document_loaders/recursive_url_loader.py b/libs/langchain/langchain/document_loaders/recursive_url_loader.py
@@ -179,10 +179,14 @@ async def _async_get_child_links_recursive(
         # Disable SSL verification because websites may have invalid SSL certificates,
         # but won't cause any security issues for us.
         close_session = session is None
-        session = session or aiohttp.ClientSession(
-            connector=aiohttp.TCPConnector(ssl=False),
-            timeout=aiohttp.ClientTimeout(total=self.timeout),
-            headers=self.headers,
+        session = (
+            session
+            if session is not None
+            else aiohttp.ClientSession(
+                connector=aiohttp.TCPConnector(ssl=False),
+                timeout=aiohttp.ClientTimeout(total=self.timeout),
+                headers=self.headers,
+            )
         )
         async with self._lock:  # type: ignore
             visited.add(url)
@@ -194,6 +198,8 @@ async def _async_get_child_links_recursive(
                 f"Unable to load {url}. Received error {e} of type "
                 f"{e.__class__.__name__}"
             )
+            if close_session:
+                await session.close()
             return []
         results = []
         content = self.extractor(text)