Rework logging (#3358)

Unified all log files into one. ### What problem does this PR solve? Unified all log files into one. ### Type of change - [x] Refactoring
infiniflow · Nov 12, 2024 · a2a5631 · a2a5631
1 parent 567a756
commit a2a5631
Show file tree

Hide file tree

Showing 75 changed files with 481 additions and 853 deletions.
diff --git a/agent/canvas.py b/agent/canvas.py
@@ -14,14 +14,12 @@
 #  limitations under the License.
 #
 import json
-import traceback
 from abc import ABC
 from copy import deepcopy
 from functools import partial
 from agent.component import component_class
 from agent.component.base import ComponentBase
-from agent.settings import flow_logger, DEBUG
-
+from api.utils.log_utils import logger
 
 class Canvas(ABC):
     """
@@ -189,7 +187,7 @@ def prepare2run(cpns):
                 if cpn.component_name == "Answer":
                     self.answer.append(c)
                 else:
-                    if DEBUG: print("RUN: ", c)
+                    logger.debug(f"Canvas.prepare2run: {c}")
                     cpids = cpn.get_dependent_components()
                     if any([c not in self.path[-1] for c in cpids]):
                         continue
@@ -199,7 +197,7 @@ def prepare2run(cpns):
 
         prepare2run(self.components[self.path[-2][-1]]["downstream"])
         while 0 <= ran < len(self.path[-1]):
-            if DEBUG: print(ran, self.path)
+            logger.debug(f"Canvas.run: {ran} {self.path}")
             cpn_id = self.path[-1][ran]
             cpn = self.get_component(cpn_id)
             if not cpn["downstream"]: break
@@ -219,7 +217,7 @@ def prepare2run(cpns):
                             self.get_component(p)["obj"].set_exception(e)
                             prepare2run([p])
                             break
-                    traceback.print_exc()
+                    logger.exception("Canvas.run got exception")
                     break
                 continue
 
@@ -231,7 +229,7 @@ def prepare2run(cpns):
                         self.get_component(p)["obj"].set_exception(e)
                         prepare2run([p])
                         break
-                traceback.print_exc()
+                logger.exception("Canvas.run got exception")
                 break
 
         if self.answer:

diff --git a/agent/component/arxiv.py b/agent/component/arxiv.py
@@ -16,9 +16,8 @@
 from abc import ABC
 import arxiv
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
-
+from api.utils.log_utils import logger
 
 class ArXivParam(ComponentParamBase):
     """
@@ -65,5 +64,5 @@ def _run(self, history, **kwargs):
             return ArXiv.be_output("")
 
         df = pd.DataFrame(arxiv_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {str(df)}")
         return df
diff --git a/agent/component/baidu.py b/agent/component/baidu.py
@@ -13,14 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import random
 from abc import ABC
-from functools import partial
 import pandas as pd
 import requests
 import re
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class BaiduParam(ComponentParamBase):
@@ -64,6 +62,6 @@ def _run(self, history, **kwargs):
             return Baidu.be_output("")
 
         df = pd.DataFrame(baidu_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {str(df)}")
         return df
 
diff --git a/agent/component/base.py b/agent/component/base.py
@@ -17,14 +17,14 @@
 import builtins
 import json
 import os
-from copy import deepcopy
 from functools import partial
-from typing import List, Dict, Tuple, Union
+from typing import Tuple, Union
 
 import pandas as pd
 
 from agent import settings
-from agent.settings import flow_logger, DEBUG
+from api.utils.log_utils import logger
+
 
 _FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"
 _DEPRECATED_PARAMS = "_deprecated_params"
@@ -361,13 +361,13 @@ def _not_in(value, wrong_value_list):
 
     def _warn_deprecated_param(self, param_name, descr):
         if self._deprecated_params_set.get(param_name):
-            flow_logger.warning(
+            logger.warning(
                 f"{descr} {param_name} is deprecated and ignored in this version."
             )
 
     def _warn_to_deprecate_param(self, param_name, descr, new_param):
         if self._deprecated_params_set.get(param_name):
-            flow_logger.warning(
+            logger.warning(
                 f"{descr} {param_name} will be deprecated in future release; "
                 f"please use {new_param} instead."
             )
@@ -403,7 +403,7 @@ def get_dependent_components(self):
         return cpnts
 
     def run(self, history, **kwargs):
-        flow_logger.info("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
+        logger.info("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
                                                               json.dumps(kwargs, ensure_ascii=False)))
         try:
             res = self._run(history, **kwargs)
@@ -463,7 +463,7 @@ def get_input(self):
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
 
-        if DEBUG: print(self.component_name, reversed_cpnts[::-1])
+        logger.debug(f"{self.component_name} {reversed_cpnts[::-1]}")
         for u in reversed_cpnts[::-1]:
             if self.get_component_name(u) in ["switch", "concentrator"]: continue
             if self.component_name.lower() == "generate" and self.get_component_name(u) == "retrieval":

diff --git a/agent/component/bing.py b/agent/component/bing.py
@@ -16,9 +16,8 @@
 from abc import ABC
 import requests
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
-
+from api.utils.log_utils import logger
 
 class BingParam(ComponentParamBase):
     """
@@ -81,5 +80,5 @@ def _run(self, history, **kwargs):
             return Bing.be_output("")
 
         df = pd.DataFrame(bing_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {str(df)}")
         return df
diff --git a/agent/component/categorize.py b/agent/component/categorize.py
@@ -17,7 +17,7 @@
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
-from agent.settings import DEBUG
+from api.utils.log_utils import logger
 
 
 class CategorizeParam(GenerateParam):
@@ -34,7 +34,7 @@ def check(self):
         super().check()
         self.check_empty(self.category_description, "[Categorize] Category examples")
         for k, v in self.category_description.items():
-            if not k: raise ValueError(f"[Categorize] Category name can not be empty!")
+            if not k: raise ValueError("[Categorize] Category name can not be empty!")
             if not v.get("to"): raise ValueError(f"[Categorize] 'To' of category {k} can not be empty!")
 
     def get_prompt(self):
@@ -77,7 +77,7 @@ def _run(self, history, **kwargs):
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": input}],
                             self._param.gen_conf())
-        if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::", input)
+        logger.debug(f"input: {input}, answer: {str(ans)}")
         for c in self._param.category_description.keys():
             if ans.lower().find(c.lower()) >= 0:
                 return Categorize.be_output(self._param.category_description[c]["to"])

diff --git a/agent/component/duckduckgo.py b/agent/component/duckduckgo.py
@@ -16,8 +16,8 @@
 from abc import ABC
 from duckduckgo_search import DDGS
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class DuckDuckGoParam(ComponentParamBase):
@@ -62,5 +62,5 @@ def _run(self, history, **kwargs):
             return DuckDuckGo.be_output("")
 
         df = pd.DataFrame(duck_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug("df: {df}")
         return df
diff --git a/agent/component/github.py b/agent/component/github.py
@@ -16,8 +16,8 @@
 from abc import ABC
 import pandas as pd
 import requests
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class GitHubParam(ComponentParamBase):
@@ -57,5 +57,5 @@ def _run(self, history, **kwargs):
             return GitHub.be_output("")
 
         df = pd.DataFrame(github_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {df}")
         return df
diff --git a/agent/component/google.py b/agent/component/google.py
@@ -16,8 +16,8 @@
 from abc import ABC
 from serpapi import GoogleSearch
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class GoogleParam(ComponentParamBase):
@@ -85,12 +85,12 @@ def _run(self, history, **kwargs):
                  "hl": self._param.language, "num": self._param.top_n})
             google_res = [{"content": '<a href="' + i["link"] + '">' + i["title"] + '</a>    ' + i["snippet"]} for i in
                           client.get_dict()["organic_results"]]
-        except Exception as e:
+        except Exception:
             return Google.be_output("**ERROR**: Existing Unavailable Parameters!")
 
         if not google_res:
             return Google.be_output("")
 
         df = pd.DataFrame(google_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {df}")
         return df
diff --git a/agent/component/googlescholar.py b/agent/component/googlescholar.py
@@ -15,9 +15,9 @@
 #
 from abc import ABC
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
 from scholarly import scholarly
+from api.utils.log_utils import logger
 
 
 class GoogleScholarParam(ComponentParamBase):
@@ -58,13 +58,13 @@ def _run(self, history, **kwargs):
                     'pub_url'] + '"></a> ' + "\n author: " + ",".join(pub['bib']['author']) + '\n Abstract: ' + pub[
                                                    'bib'].get('abstract', 'no abstract')})
 
-            except StopIteration or Exception as e:
-                print("**ERROR** " + str(e))
+            except StopIteration or Exception:
+                logger.exception("GoogleScholar")
                 break
 
         if not scholar_res:
             return GoogleScholar.be_output("")
 
         df = pd.DataFrame(scholar_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {df}")
         return df
diff --git a/agent/component/keyword.py b/agent/component/keyword.py
@@ -18,7 +18,7 @@
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
-from agent.settings import DEBUG
+from api.utils.log_utils import logger
 
 
 class KeywordExtractParam(GenerateParam):
@@ -58,5 +58,5 @@ def _run(self, history, **kwargs):
                             self._param.gen_conf())
 
         ans = re.sub(r".*keyword:", "", ans).strip()
-        if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::")
+        logger.info(f"ans: {ans}")
         return KeywordExtract.be_output(ans)
diff --git a/agent/component/pubmed.py b/agent/component/pubmed.py
@@ -18,8 +18,8 @@
 import re
 import pandas as pd
 import xml.etree.ElementTree as ET
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class PubMedParam(ComponentParamBase):
@@ -65,5 +65,5 @@ def _run(self, history, **kwargs):
             return PubMed.be_output("")
 
         df = pd.DataFrame(pubmed_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {df}")
         return df
diff --git a/agent/component/relevant.py b/agent/component/relevant.py
@@ -18,6 +18,7 @@
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
 from rag.utils import num_tokens_from_string, encoder
+from api.utils.log_utils import logger
 
 
 class RelevantParam(GenerateParam):
@@ -70,7 +71,7 @@ def _run(self, history, **kwargs):
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": ans}],
                             self._param.gen_conf())
 
-        print(ans, ":::::::::::::::::::::::::::::::::")
+        logger.info(ans)
         if ans.lower().find("yes") >= 0:
             return Relevant.be_output(self._param.yes)
         if ans.lower().find("no") >= 0:

diff --git a/agent/component/retrieval.py b/agent/component/retrieval.py
@@ -22,6 +22,7 @@
 from api.db.services.llm_service import LLMBundle
 from api.settings import retrievaler
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class RetrievalParam(ComponentParamBase):
@@ -80,7 +81,7 @@ def _run(self, history, **kwargs):
         df = pd.DataFrame(kbinfos["chunks"])
         df["content"] = df["content_with_weight"]
         del df["content_with_weight"]
-        print(">>>>>>>>>>>>>>>>>>>>>>>>>>\n", query, df)
+        logger.debug("{} {}".format(query, df))
         return df
 
 
diff --git a/agent/component/rewrite.py b/agent/component/rewrite.py
@@ -17,6 +17,7 @@
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
+from api.utils.log_utils import logger
 
 
 class RewriteQuestionParam(GenerateParam):
@@ -104,7 +105,7 @@ def _run(self, history, **kwargs):
         self._canvas.history.pop()
         self._canvas.history.append(("user", ans))
 
-        print(ans, ":::::::::::::::::::::::::::::::::")
+        logger.info(ans)
         return RewriteQuestion.be_output(ans)
 
 
diff --git a/agent/component/wikipedia.py b/agent/component/wikipedia.py
@@ -13,13 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import random
 from abc import ABC
-from functools import partial
 import wikipedia
 import pandas as pd
-from agent.settings import DEBUG
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.log_utils import logger
 
 
 class WikipediaParam(ComponentParamBase):
@@ -65,5 +63,5 @@ def _run(self, history, **kwargs):
             return Wikipedia.be_output("")
 
         df = pd.DataFrame(wiki_res)
-        if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+        logger.debug(f"df: {df}")
         return df