diff --git a/docs/create_robots_whoami.md b/docs/create_robots_whoami.md index 08a2946f2..e255eb635 100644 --- a/docs/create_robots_whoami.md +++ b/docs/create_robots_whoami.md @@ -6,6 +6,8 @@ To configure RAI for your robot, provide contents for your robot's so called `wh Your robot's `whoami` package serves as a configuration package for the `rai_whoami` node. +> **TIP**: The Human-Machine Interface (HMI), both text and voice versions, relies heavily on the whoami package. It uses the robot's identity, constitution, and documentation to provide context-aware responses and ensure the robot behaves according to its defined characteristics. + ## Example (Franka Emika Panda arm) 1. Setup the repository using 1st and 2nd step from [Setup](../README.md#setup) @@ -16,10 +18,10 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who poetry run create_rai_ws --name panda --destination-directory src/examples ``` -3. Fill in the `src/examples/panda_whoami/description` folder with data:\ - 2.1 Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images`\ - 2.2 Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` - 2.3 Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` +3. Fill in the `src/examples/panda_whoami/description` folder with data: + 3.1 Save [this image](https://robodk.com/robot/img/Franka-Emika-Panda-robot.png) into `src/examples/panda_whoami/description/images` + 3.2 Save [this document](https://github.com/user-attachments/files/16417196/Franka.Emika.Panda.robot.-.RoboDK.pdf) in `src/examples/panda_whoami/description/documentation` + 3.3 Save [this urdf](https://github.com/frankaemika/franka_ros/blob/develop/franka_description/robots/panda/panda.urdf.xacro) in `src/examples/panda_whoami/description/urdf` 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity. @@ -27,14 +29,23 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who > [config.toml](../config.toml) (`ollama` works locally, see [docs/vendors.md](./vendors.md#ollama)). ```shell -poetry run parse_whoami_package src/examples/panda_whoami/description +poetry run parse_whoami_package src/examples/panda_whoami ``` +5. Optional: Examine the generated files + +After running the `parse_whoami_package` command, you can inspect the generated files in the `src/examples/panda_whoami/description/generated` directory. These files contain important information about your robot: + + - `robot_identity.txt`: Contains a detailed description of the robot's identity, capabilities, and characteristics. + - `robot_description.urdf.txt`: Provides a summary of the robot's URDF (Unified Robot Description Format), describing its physical structure. + - `robot_constitution.txt`: Outlines the ethical guidelines and operational rules for the robot. + - `faiss_index`: A directory containing the vector store of the robot's documentation, used for efficient information retrieval. + ## Testing You can test your new `panda_whoami` package by calling `rai_whoami` services: -2. Building the `rai_whoami` package and running the `rai_whoami_node` for your `Panda` robot: +1. Building the `rai_whoami` package and running the `rai_whoami_node` for your `Panda` robot: ```shell colcon build --symlink-install diff --git a/src/rai/rai/cli/rai_cli.py b/src/rai/rai/cli/rai_cli.py index e302ca0c0..c7e00623a 100644 --- a/src/rai/rai/cli/rai_cli.py +++ b/src/rai/rai/cli/rai_cli.py @@ -38,24 +38,20 @@ def parse_whoami_package(): description="Parse robot whoami package. Script builds a vector store, creates a robot identity and a URDF description." ) parser.add_argument( - "documentation_root", type=str, help="Path to the root of the documentation" - ) - parser.add_argument( - "--output", - type=str, - required=False, - default=None, - help="Path to the output directory", + "whoami_package_root", type=str, help="Path to the root of the whoami package" ) + args = parser.parse_args() - save_dir = args.output if args.output is not None else args.documentation_root + save_dir = Path(args.whoami_package_root) / "description" / "generated" + description_path = Path(args.whoami_package_root) / "description" + save_dir.mkdir(parents=True, exist_ok=True) llm = get_llm_model(model_type="simple_model") embeddings_model = get_embeddings_model() def calculate_urdf_tokens(): combined_urdf = "" - xacro_files = glob.glob(args.documentation_root + "/urdf/*.xacro") + xacro_files = glob.glob(str(description_path / "urdf" / "*.xacro")) for xacro_file in xacro_files: combined_urdf += f"# {xacro_file}\n" combined_urdf += open(xacro_file, "r").read() @@ -65,18 +61,23 @@ def calculate_urdf_tokens(): def build_urdf_description(): logger.info("Building the URDF description...") combined_urdf = "" - xacro_files = glob.glob(args.documentation_root + "/urdf/*.xacro") + xacro_files = glob.glob(str(description_path / "urdf" / "*.xacro")) for xacro_file in xacro_files: combined_urdf += f"# {xacro_file}\n" combined_urdf += open(xacro_file, "r").read() combined_urdf += "\n\n" prompt = "You will be given a URDF file. Your task is to create a short and detailed description of links and joints. " - parsed_urdf = llm.invoke( - [SystemMessage(content=prompt), HumanMessage(content=str(combined_urdf))] - ).content - - with open(save_dir + "/robot_description.urdf.txt", "w") as f: + parsed_urdf = "" + if len(combined_urdf): + parsed_urdf = llm.invoke( + [ + SystemMessage(content=prompt), + HumanMessage(content=str(combined_urdf)), + ] + ).content + + with open(str(save_dir / "robot_description.urdf.txt"), "w") as f: f.write(parsed_urdf) logger.info("Done") @@ -84,7 +85,7 @@ def build_docs_vector_store(): logger.info("Building the robot docs vector store...") faiss_index = FAISS.from_documents(docs, embeddings_model) faiss_index.add_documents(docs) - faiss_index.save_local(save_dir) + faiss_index.save_local(str(save_dir)) def build_robot_identity(): logger.info("Building the robot identity...") @@ -98,7 +99,7 @@ def build_robot_identity(): "Your reply should start with I am a ..." ) - images = glob.glob(args.documentation_root + "/images/*") + images = glob.glob(str(description_path / "images" / "*")) messages = [SystemMessage(content=prompt)] + [ HumanMultimodalMessage( @@ -109,12 +110,12 @@ def build_robot_identity(): output = llm.invoke(messages) assert isinstance(output.content, str), "Malformed output" - with open(save_dir + "/robot_identity.txt", "w") as f: + with open(str(save_dir / "robot_identity.txt"), "w") as f: f.write(output.content) logger.info("Done") docs = ingest_documentation( - documentation_root=args.documentation_root + "/documentation" + documentation_root=str(description_path / "documentation") ) documentation = str([doc.page_content for doc in docs]) n_tokens = len(documentation) // 4.0 @@ -189,7 +190,8 @@ def create_rai_ws(): (package_path / "documentation").mkdir(exist_ok=True) (package_path / "images").mkdir(exist_ok=True) - (package_path / "robot_constitution.txt").touch() + (package_path / "generated").mkdir(exist_ok=True) + (package_path / "generated" / "robot_constitution.txt").touch() default_constitution_path = ( "src/rai/rai/cli/resources/default_robot_constitution.txt" @@ -197,7 +199,7 @@ def create_rai_ws(): with open(default_constitution_path, "r") as file: default_constitution = file.read() - with open(f"{package_path}/robot_constitution.txt", "w") as file: + with open(f"{package_path}/generated/robot_constitution.txt", "w") as file: file.write(default_constitution) # Modify setup.py file diff --git a/src/rai_whoami/rai_whoami/rai_whoami_node.py b/src/rai_whoami/rai_whoami/rai_whoami_node.py index ca5de21a5..07d7c5584 100644 --- a/src/rai_whoami/rai_whoami/rai_whoami_node.py +++ b/src/rai_whoami/rai_whoami/rai_whoami_node.py @@ -67,7 +67,7 @@ def __init__(self): ) # type: ignore self.robot_constitution_path = os.path.join( get_package_share_directory(self.robot_description_package), - "description/robot_constitution.txt", + "description/generated/robot_constitution.txt", ) with open(self.robot_constitution_path, "r") as file: @@ -83,7 +83,7 @@ def __init__(self): def _load_documentation(self) -> FAISS: faiss_index = FAISS.load_local( get_package_share_directory(self.robot_description_package) - + "/description", + + "/description/generated", get_embeddings_model(), allow_dangerous_deserialization=True, ) @@ -95,7 +95,7 @@ def get_urdf_callback( """Return URDF description""" urdf_path = ( get_package_share_directory(self.robot_description_package) - + "/description/robot_description.urdf.txt" + + "/description/generated/robot_description.urdf.txt" ) with open(urdf_path, "r") as f: urdf = f.read() @@ -175,7 +175,7 @@ def get_identity_callback( """Return robot identity""" identity_path = ( get_package_share_directory(self.robot_description_package) - + "/description/robot_identity.txt" + + "/description/generated/robot_identity.txt" ) with open(identity_path, "r") as f: identity = f.read()