feat(rosbot): add dedicated nav2 tools

config.toml

@@ -1,5 +1,7 @@
 [vendor]
-name = "openai" # openai, aws, ollama
+simple_model = "openai" # openai, aws, ollama
+embeddings_model = "ollama"
+complex_model = "openai"
 
 [aws]
 simple_model = "anthropic.claude-3-haiku-20240307-v1:0"
@@ -13,9 +15,9 @@ complex_model = "gpt-4o-2024-08-06"
 embeddings_model = "text-embedding-ada-002"
 
 [ollama]
-simple_model = "llama3.1"
+simple_model = "llama3.2"
 complex_model = "llama3.1:70b"
-embeddings_model = "llama3.1"
+embeddings_model = "llama3.2"
 base_url = "http://localhost:11434"
 
 [tracing]

docs/create_robots_whoami.md

@@ -16,7 +16,7 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who
 2. Create a whoami package for Panda
 
    ```shell
-   poetry run create_rai_ws --name panda --destination-directory src/examples
+   ./scripts/create_rai_ws.sh --name panda --destination-directory src/examples
    ```
 
 3. Fill in the `src/examples/panda_whoami/description` folder with data:
@@ -30,11 +30,12 @@ Your robot's `whoami` package serves as a configuration package for the `rai_who
 4. Run the `parse_whoami_package`. This will process the documentation, building it into a vector database, which is used by RAI agent to reason about its identity.
 
 > [!IMPORTANT]
-> Parsing bigger documents might lead to costs. Embedding model can be configured in
+> Parsing bigger documents with Cloud vendors might lead to costs. Consider using the
+> local `ollama` provider for this task. Embedding model can be configured in
 > [config.toml](../config.toml) (`ollama` works locally, see [docs/vendors.md](./vendors.md#ollama)).
 
 ```shell
-poetry run parse_whoami_package src/examples/panda_whoami
+./scripts/parse_whoami_package.sh src/examples/panda_whoami
 ```
 
 5. Optional: Examine the generated files

examples/rosbot-xl-generic-node-demo.py

@@ -16,25 +16,23 @@
 
 import rclpy
 import rclpy.executors
-from rai_open_set_vision.tools import GetDetectionTool, GetDistanceToObjectsTool
+from rai_open_set_vision import GetDetectionTool
 
 from rai.node import RaiStateBasedLlmNode
+from rai.tools.ros.manipulation import GetObjectPositionsTool
 from rai.tools.ros.native import (
     GetCameraImage,
-    GetMsgFromTopic,
     Ros2PubMessageTool,
     Ros2ShowMsgInterfaceTool,
 )
-
-# from rai.tools.ros.native_actions import Ros2RunActionSync
 from rai.tools.ros.native_actions import (
+    GetTransformTool,
     Ros2CancelAction,
     Ros2GetActionResult,
     Ros2GetLastActionFeedback,
     Ros2IsActionComplete,
     Ros2RunActionAsync,
 )
-from rai.tools.ros.tools import GetCurrentPositionTool
 from rai.tools.time import WaitForSecondsTool
 
 
@@ -50,11 +48,13 @@ def main():
     topics_whitelist = [
         "/rosout",
         "/camera/camera/color/image_raw",
+        "/camera/camera/color/camera_info",
         "/camera/camera/depth/image_rect_raw",
+        "/camera/camera/depth/camera_info",
         "/map",
         "/scan",
         "/diagnostics",
-        "/cmd_vel",
+        # "/cmd_vel",
         "/led_strip",
     ]
 
@@ -81,6 +81,7 @@ def main():
 
     <rule> use /cmd_vel topic very carefully. Obstacle detection works only with nav2 stack, so be careful when it is not used. </rule>>
     <rule> be patient with running ros2 actions. usually the take some time to run. </rule>
+    <rule> Always check your transform before and after you perform ros2 actions, so that you can verify if it worked. </rule>
 
     Navigation tips:
     - it's good to start finding objects by rotating, then navigating to some diverse location with occasional rotations. Remember to frequency detect objects.
@@ -136,12 +137,20 @@ def main():
             Ros2GetActionResult,
             Ros2GetLastActionFeedback,
             Ros2ShowMsgInterfaceTool,
-            GetCurrentPositionTool,
             WaitForSecondsTool,
-            GetMsgFromTopic,
             GetCameraImage,
             GetDetectionTool,
-            GetDistanceToObjectsTool,
+            GetTransformTool,
+            (
+                GetObjectPositionsTool,
+                dict(
+                    target_frame="odom",
+                    source_frame="sensor_frame",
+                    camera_topic="/camera/camera/color/image_raw",
+                    depth_topic="/camera/camera/depth/image_rect_raw",
+                    camera_info_topic="/camera/camera/color/camera_info",
+                ),
+            ),
         ],
     )
 

pyproject.toml

@@ -86,7 +86,3 @@ markers = [
     "billable: marks test as billable  (deselect with '-m \"not billable\"')",
 ]
 addopts = "-m 'not billable' --ignore=src"
-
-[tool.poetry.scripts]
-create_rai_ws = "rai.cli.rai_cli:create_rai_ws"
-parse_whoami_package = "rai.cli.rai_cli:parse_whoami_package"

scripts/create_rai_ws.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+python -c "from rai.cli.rai_cli import create_rai_ws; create_rai_ws()" "$@"

scripts/parse_whoami_package.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+python -c "from rai.cli.rai_cli import parse_whoami_package; parse_whoami_package()" "$@"

src/examples/turtlebot4/README.md

@@ -0,0 +1,217 @@
+# Turtlebot4 tutorial
+
+The following tutorial will help you get started with [rai](https://github.com/RobotecAI/rai)
+using Turtlebot4 simulation.
+
+## Recording
+
+[link](https://robotecai-my.sharepoint.com/:v:/g/personal/bartlomiej_boczek_robotec_ai/EfPTiZscCTROtmtoyHv_ykIBFKN5qh1pecxfLmLI6I4QeA?nav=eyJyZWZlcnJhbEluZm8iOnsicmVmZXJyYWxBcHAiOiJPbmVEcml2ZUZvckJ1c2luZXNzIiwicmVmZXJyYWxBcHBQbGF0Zm9ybSI6IldlYiIsInJlZmVycmFsTW9kZSI6InZpZXciLCJyZWZlcnJhbFZpZXciOiJNeUZpbGVzTGlua0NvcHkifX0&e=0QJnGk)
+
+## Step by step instructions
+
+0. Install nav2 packates
+
+   ```shell
+   sudo apt install "ros-${ROS_DISTRO}-nav2-*"
+
+   # add to ~/.bashrc
+   . /opt/ros/humble/setup.bash
+   # or
+   . ./opt/ros/jazzy/setup.bash
+   export RMW_IMPLEMENTATION=rmw_fastrtps_cpp
+
+   ```
+
+1. Clone and install [rai](https://github.com/RobotecAI/rai)
+
+   1. Follow steps from [1. Setting up the workspace](https://github.com/RobotecAI/rai?tab=readme-ov-file#1-setting-up-the-workspace)
+   2. Then follow steps from [2. Build the project](https://github.com/RobotecAI/rai?tab=readme-ov-file#2-build-the-project)
+
+2. O3DE simulation setup
+
+   1. Download turtlebot4 simulation binary matching your platform from :
+
+      - Ubuntu 22.04 & ros2 humble: [link](https://robotecai-my.sharepoint.com/:u:/g/personal/bartlomiej_boczek_robotec_ai/EZLmGtPNgl9Kiu4royJJnVgB5tjS2Vze0myXDyVJtNcnRw?e=L42Z4z)
+      - Ubuntu 24.04 & ros2 jazzy: [link](https://robotecai-my.sharepoint.com/:u:/g/personal/bartlomiej_boczek_robotec_ai/ETkT-jvozlpBtuG1piDeqggBRmWl5eylIChc_g0v_EetpA?e=EjcJqe)
+
+      > [!NOTE]  
+      > If you would like to make changes to the simulation and create your own binary please follow this [README.md](https://github.com/RobotecAI/ROSCon2024Tutorial/README.md)
+
+   2. Unzip it:
+
+      ```bash
+      cd rai
+      # for Ubuntu 22.04 jammy and ros2 humble
+      unzip -d src/examples/turtlebot4/simulation Turtlebot4_jammyhumble_0.0.1.zip
+
+      # for Ubuntu 24.04 noble and ros2 jazzy
+      unzip -d src/examples/turtlebot4/simulation Turtlebot4_noblejazzy_0.0.1.zip
+      ```
+
+3. Setup your LLM vendor: [docs/vendors.md](../../../docs/vendors.md). OpenAI or
+   AWS Bedrock are recommended for models, since current local `ollama` models don't
+   support vision & tool calling.
+   For `embeddings_model` `ollama` vendor works well.
+
+4. Configure `rai_whoami_node` (based on ["Your robot identity in RAI"](https://github.com/RobotecAI/rai/blob/development/docs/create_robots_whoami.md) tutorial):
+> [!TIP]
+> Generated files can be downloaded from [here](https://robotecai-my.sharepoint.com/:u:/g/personal/bartlomiej_boczek_robotec_ai/EbPZSEdXYaRGoeecu6oJg6QBsI4ZOe_mrU3uOtOflnIjQg?e=HX8ZHB) unzip them to `src/examples/turtlebot4_whoami/description/generated` with a command:
+> `unzip -d src/examples/turtlebot4_whoami/description turtlebot4_whoami_generated.zip`
+   1. Create `whoami` package for turtlebot4 in `src/examples/turtlebot4`
+
+      ```bash
+      ./scripts/create_rai_ws.sh --name turtlebot4 --destination-directory src/examples
+      ```
+
+   2. Download official turtlebot4 [data sheet](https://bit.ly/3KCp3Du) into
+      `src/examples/turtlebot4_whoami/description/documentation`
+   3. Download [image](https://s3.amazonaws.com/assets.clearpathrobotics.com/wp-content/uploads/2022/03/16113604/Turtlebot-4-20220207.44.png) of turtlebot4 into `src/examples/turtlebot4_whoami/description/images`
+   4. Create robot's identity. Run the `parse_whoami_package`. This will process the documentation, building
+      it into a vector database, which is used by RAI agent to reason about its identity.
+      ```bash
+      ./scripts/parse_whoami_package.sh src/examples/turtlebot4_whoami
+      # you will be asked to press `y` to continue
+      ```
+
+   5. Build the workspace which now includes the new package
+
+      ```bash
+      deactivate # if poetry env is activated
+      colcon build --symlink-install
+      ```
+
+   6. Ensure `turtlebot4_whoami` package has been built:
+
+      ```bash
+      . ./setup_shell.sh
+      ros2 pkg list | grep turtlebot4
+      ```
+
+   Congratulations! Your `rai_whoami_node` is configured. In the following steps
+   your RAI agent will assume a turtlebot4 "personality".
+
+5. Run rai agent:
+
+   ```bash
+   ros2 launch ./src/examples/turtlebot4/turtlebot.launch.xml \
+       game_launcher:=./src/examples/turtlebot4/simulation/TurtleBot4DemoGamePackage/TurtleBot4Demo.GameLauncher
+   ```
+
+   You should be able to see a simulation scene.
+
+   > **Tip**: Press 1, 2 or 3 on the keyboard when simulation window to change the
+   > camera. Use W,A,S,D to move the camera.
+
+6. Open you internet browser and go to `localhost:8501`
+
+7. You can interact with your RAI agent through the chat. On the left you can communicate
+   with the RAI HMI Agent. On the right you will see status of missions that were send
+   to the RAI Node and are executed on your robot.
+
+   - `HMI Agent` is responsible for human-robot interaction.
+   - `RAI Node` is responsible for executing tasks on the robot.
+
+   For example you can try such prompts:
+
+   - who are you? (testing robot's identity)
+   - what is the voltage of the batery? (use RAG: query vector database for the answer)
+   - could you bring me something from the kitchen? (testing robot's identity)
+   - what are your ros2 interfaces? (discovery of ros2 interfaces)
+   - tell me what you see (interpretation of camera image)
+   - spin yourself left by 45 degrees (interaction with the robot using ros2 interfaces) - table with the robotic are shoud be visible in the camera
+   - use robotic arm to pick up a box from the table (identity and intefaces doesn't allow it)
+   - drive towards the table (when table is visible, testing ability to interpret camera image and perform actions based on the knowledge)
+
+### Troubleshooting
+
+_My robot doesn't have an identity._
+
+1. Query `rai_whoami_node` for the identity manually
+
+   ```bash
+   ros2 service call /rai_whoami_identity_service std_srvs/srv/Trigger
+   ```
+
+2. Make sure all required files were generated correctly in the `turtlebot4_whoami` package and have similar sizes to those listed below.
+
+   ```bash
+   ls -sh src/examples/turtlebot4_whoami/description/
+   total 52K
+   4.0K documentation  4.0K images   28K index.faiss  8.0K index.pkl  4.0K robot_constitution.txt  4.0K robot_identity.txt
+   ```
+
+   You can also check the contents of `robot_indentify.txt` file (it is generated by LLM, but should be simillar to the one below).
+
+   ```bash
+   cat src/examples/turtlebot4_whoami/description/robot_identity.txt
+   I am a TurtleBot® 4, a robotics learning platform designed for education and
+   research in robotics. This next-generation mobile robot is built on the iRobot®
+   Create® 3 mobile base and comes fully assembled with ROS 2 installed and configured,
+   making it accessible for beginners and experienced developers alike.
+   ...
+   ```
+
+   If files above are incorrect please check your vendor configuration is as described
+   in [docs/vendors.md](../../../docs/vendors.md) and rerun point 4 of this tutorial.
+
+---
+
+_Robot doesn't move in the simulation._
+
+1. Make sure you can see ros2 topic simulation binary is running
+
+   1. Run the binary manually:
+
+      ```bash
+      cd src/examples/turtlebot4/simulation/
+      ./TurtleBot4DemoGamePackage/TurtleBot4Demo.GameLauncher -bg_ConnectToAssetProcessor=0
+      ```
+
+   2. Check ros2 topics:
+
+      ```bash
+      ros2 topic list
+      ```
+
+   3. Verify if Turtlebot simulation binary publishes such topics:
+
+      ```
+      /camera_color_info
+      /camera_image_color
+      /clock
+      /cmd_vel
+      /parameter_events
+      /rosout
+      /scan
+      /tf
+      ```
+
+2. Make sure navigation stack actions work correctly:
+
+   1. Run binary as in the step above.
+   2. Run navigation stack:
+
+      ```bash
+      cd src/examples/turtlebot4
+      ./run-nav.bash
+      ```
+
+   3. Check available actions:
+
+      ```bash
+      ros2 action list
+      ```
+
+   4. Verify if such actions are available:
+
+      ```
+      /backup
+      /navigate_to_pose
+      /spin
+      ```
+
+   5. Try to run one of them from command line:
+
+      ```bash
+      ros2 action send_goal /spin nav2_msgs/action/Spin "{target_yaw: 3.14}"
+      ```

src/examples/turtlebot4/allowlist.txt

@@ -0,0 +1,10 @@
+/cmd_vel
+/rosout
+/map
+/odom
+/camera_image_color
+/backup
+/drive_on_heading
+/navigate_through_poses
+/navigate_to_pose
+/spin