chore: cleanup (#149)

Co-authored-by: Kajetan Rachwał <[email protected]>

.github/workflows/poetry-test.yml

@@ -22,7 +22,7 @@ jobs:
         uses: snok/install-poetry@v1
 
       - name: Install python dependencies
-        run: poetry install
+        run: poetry install --with gdino
 
       - name: Install ROS 2 dependencies
         shell: bash

docs/multimodal_messages.md

@@ -17,7 +17,7 @@ class ToolMultimodalMessage(ToolMessage, MultimodalMessage):
 Example:
 
 ```python
-from rai.scenario_engine.messages import HumanMultimodalMessage, preprocess_image
+from rai.messages import HumanMultimodalMessage, preprocess_image
 from langchain_openai.chat_models import ChatOpenAI
 
 base64_image = preprocess_image('https://raw.githubusercontent.com/RobotecAI/RobotecGPULidar/develop/docs/image/rgl-logo.png')

pyproject.toml

@@ -46,6 +46,7 @@ pypdf = "^4.2.0"
 langchain-ollama = "^0.1.1"
 
 streamlit = "^1.37.1"
+deprecated = "^1.2.14"
 [tool.poetry.group.dev.dependencies]
 ipykernel = "^6.29.4"
 

src/rai/package.xml

@@ -13,6 +13,10 @@
   <test_depend>ament_pep257</test_depend>
   <test_depend>python3-pytest</test_depend>
 
+  <exec_depend>nav2_msgs</exec_depend>
+  <exec_depend>nav2_simple_commander</exec_depend>
+  <exec_depend>tf_transformations</exec_depend>
+
   <export>
     <build_type>ament_python</build_type>
   </export>

src/rai/rai/agents/state_based.py

@@ -51,7 +51,7 @@
 from langgraph.utils import RunnableCallable
 from rclpy.impl.rcutils_logger import RcutilsLogger
 
-from rai.scenario_engine.messages import (
+from rai.messages import (
     HumanMultimodalMessage,
     MultimodalArtifact,
     ToolMultimodalMessage,

src/rai/rai/apps/talk_to_docs.py

@@ -30,7 +30,7 @@
 from langchain_openai import OpenAIEmbeddings
 from langgraph.graph import StateGraph
 
-from rai.documents.loader import ingest_documentation
+from rai.apps.document_loader import ingest_documentation
 
 logging.basicConfig(level=logging.WARN)
 

src/rai/rai/cli/rai_cli.py

@@ -23,7 +23,8 @@
 from langchain_openai import ChatOpenAI, OpenAIEmbeddings
 
 from rai.apps.talk_to_docs import ingest_documentation
-from rai.scenario_engine.messages import HumanMultimodalMessage, preprocess_image
+from rai.messages import preprocess_image
+from rai.messages.multimodal import HumanMultimodalMessage
 
 
 def parse_whoami_package():

src/rai_hmi/rai_hmi/custom_mavigator.py → src/rai/rai/extensions/navigator.py

@@ -1,3 +1,18 @@
+# Copyright (C) 2024 Robotec.AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import rclpy
 from builtin_interfaces.msg import Duration
 from geometry_msgs.msg import Point

src/rai/rai/communication/__init__.py → src/rai/rai/messages/__init__.py

@@ -13,7 +13,20 @@
 # limitations under the License.
 #
 
-from .communication import EmailSender
-from .ros_communication import SingleImageGrabber
+from .multimodal import (
+    AiMultimodalMessage,
+    HumanMultimodalMessage,
+    MultimodalArtifact,
+    SystemMultimodalMessage,
+    ToolMultimodalMessage,
+)
+from .utils import preprocess_image
 
-__all__ = ["EmailSender", "SingleImageGrabber"]
+__all__ = [
+    "HumanMultimodalMessage",
+    "AiMultimodalMessage",
+    "SystemMultimodalMessage",
+    "ToolMultimodalMessage",
+    "MultimodalArtifact",
+    "preprocess_image",
+]

src/rai/rai/scenario_engine/messages.py → src/rai/rai/messages/multimodal.py

@@ -13,11 +13,8 @@
 # limitations under the License.
 #
 
-import base64
-from typing import Any, Callable, Dict, List, Literal, Optional, TypedDict, Union
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
-import numpy as np
-import requests
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
 from langchain_core.messages.base import BaseMessage, get_msg_title_repr
 from langchain_core.tools import BaseTool
@@ -164,28 +161,3 @@ def __init__(
             if not any([tool.__class__.__name__ == stop_tool for tool in tools]):
                 raise ValueError("Stop tool not in tools")
         self.tools: List[BaseTool] = tools
-
-
-def preprocess_image(
-    image: Union[str, bytes, np.ndarray[Any, np.dtype[np.uint8]]],
-    encoding_function: Callable[[Any], str] = lambda x: base64.b64encode(x).decode(
-        "utf-8"
-    ),
-) -> str:
-    if isinstance(image, str) and image.startswith(("http://", "https://")):
-        response = requests.get(image)
-        response.raise_for_status()
-        image_data = response.content
-    elif isinstance(image, str):
-        with open(image, "rb") as image_file:
-            image_data = image_file.read()
-    elif isinstance(image, bytes):
-        image_data = image
-        encoding_function = lambda x: x.decode("utf-8")
-    elif isinstance(image, np.ndarray):  # type: ignore
-        image_data = image.tobytes()
-        encoding_function = lambda x: base64.b64encode(x).decode("utf-8")
-    else:
-        image_data = image
-
-    return encoding_function(image_data)

src/rai/rai/messages/utils.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2024 Robotec.AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import base64
+from typing import Any, Callable, Union
+
+import numpy as np
+import requests
+
+
+def preprocess_image(
+    image: Union[str, bytes, np.ndarray[Any, np.dtype[np.uint8]]],
+    encoding_function: Callable[[Any], str] = lambda x: base64.b64encode(x).decode(
+        "utf-8"
+    ),
+) -> str:
+    if isinstance(image, str) and image.startswith(("http://", "https://")):
+        response = requests.get(image)
+        response.raise_for_status()
+        image_data = response.content
+    elif isinstance(image, str):
+        with open(image, "rb") as image_file:
+            image_data = image_file.read()
+    elif isinstance(image, bytes):
+        image_data = image
+        encoding_function = lambda x: x.decode("utf-8")
+    elif isinstance(image, np.ndarray):  # type: ignore
+        image_data = image.tobytes()
+        encoding_function = lambda x: base64.b64encode(x).decode("utf-8")
+    else:
+        image_data = image
+
+    return encoding_function(image_data)

src/rai/rai/node.py

@@ -48,9 +48,9 @@
 from std_srvs.srv import Trigger
 
 from rai.agents.state_based import State
-from rai.communication.ros_communication import wait_for_message
-from rai.scenario_engine.messages import HumanMultimodalMessage
+from rai.messages.multimodal import HumanMultimodalMessage
 from rai.tools.ros.utils import convert_ros_img_to_base64, import_message_from_str
+from rai.tools.utils import wait_for_message
 
 
 class RosoutBuffer: