For security.

KevinHuSh · Nov 26, 2024 · 3bc354b · 3bc354b
1 parent 32cf566
commit 3bc354b
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/agent/component/crawler.py b/agent/component/crawler.py
@@ -17,6 +17,7 @@
 import asyncio
 from crawl4ai import AsyncWebCrawler
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.web_utils import is_valid_url
 
 
 class CrawlerParam(ComponentParamBase):
@@ -39,7 +40,7 @@ class Crawler(ComponentBase, ABC):
     def _run(self, history, **kwargs):
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
-        if not ans:
+        if not is_valid_url(ans):
             return Crawler.be_output("")
         try:
             result = asyncio.run(self.get_web(ans))