From cf5807804283baeb5a67daa37c816c209f16dd43 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Tue, 1 Aug 2023 17:16:00 -0700 Subject: [PATCH 01/10] made changes to be able to subscribe to Float messages --- feedingwebapp/src/ros/TestROS.jsx | 10 ++++++++-- feedingwebapp/src/ros/TestROSSubscribe.jsx | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/feedingwebapp/src/ros/TestROS.jsx b/feedingwebapp/src/ros/TestROS.jsx index df970e23..71df132e 100644 --- a/feedingwebapp/src/ros/TestROS.jsx +++ b/feedingwebapp/src/ros/TestROS.jsx @@ -45,9 +45,15 @@ function TestROS() {
{/** - * Allow users to subscribe to a topic and display its data + * Allow users to subscribe to a topic (of type std_msgs/String) and display its data */} - + +
+ + {/** + * Allow users to subscribe to a topic (of type std_msgs/Float32) and display its data + */} +
{/** diff --git a/feedingwebapp/src/ros/TestROSSubscribe.jsx b/feedingwebapp/src/ros/TestROSSubscribe.jsx index 5f9ef281..2c5454fd 100644 --- a/feedingwebapp/src/ros/TestROSSubscribe.jsx +++ b/feedingwebapp/src/ros/TestROSSubscribe.jsx @@ -1,5 +1,7 @@ // React imports import React, { useState } from 'react' +// Component +import PropTypes from 'prop-types' // Local imports import { useROS, subscribeToROSTopic } from './ros_helpers' @@ -8,7 +10,10 @@ import { useROS, subscribeToROSTopic } from './ros_helpers' * The TestROSSubscribe component demonstrates the functionality of subscribing * to a ROS topic. */ -function TestROSSubscribe() { +function TestROSSubscribe(props) { + // get the value of props + let topicType = props.topicType + // The defaults to use on this page let defaultTopicName = 'test_topic' @@ -21,7 +26,7 @@ function TestROSSubscribe() { let [recvData, setRecvData] = useState('No message received yet.') // Callback function for when the user clicks the "Subscribe" button - function subscribeTopic(event) { + function subscribeTopic(event, topicType) { // Prevent the browser from reloading the page event.preventDefault() @@ -34,7 +39,7 @@ function TestROSSubscribe() { } // Subscribe to the topic - subscribeToROSTopic(ros, topicName, 'std_msgs/String', callback) + subscribeToROSTopic(ros, topicName, topicType, callback) } // Render the component @@ -43,8 +48,8 @@ function TestROSSubscribe() { {/** * Allow users to subscribe to a topic and display its data */} -

Subscribe to a 'std_msgs/String' Topic and Display Its Data:

-
+

Subscribe to a '{topicType}' Topic and Display Its Data:

+ subscribeTopic(e, topicType)}> Topic Name:
@@ -55,5 +60,8 @@ function TestROSSubscribe() { ) } +TestROSSubscribe.propTypes = { + topicType: PropTypes.string.isRequired +} export default TestROSSubscribe From 37bd34896bc0dabb8ff922f0a6bb97d9f7fb0134 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Thu, 3 Aug 2023 18:42:27 -0700 Subject: [PATCH 02/10] [IP] Is able to listen to the FoF topic --- feedingwebapp/TechDocumentation.md | 2 +- feedingwebapp/src/Pages/Constants.js | 7 +++++ .../src/Pages/Home/MealStates/BiteDone.jsx | 28 +++++++++++++++++-- feedingwebapp/src/ros/ros_helpers.js | 10 +++++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/feedingwebapp/TechDocumentation.md b/feedingwebapp/TechDocumentation.md index acdfb374..4518f7fe 100644 --- a/feedingwebapp/TechDocumentation.md +++ b/feedingwebapp/TechDocumentation.md @@ -11,7 +11,7 @@ All functions to interact with ROS are defined in [`ros_helpers.jsx`](https://github.com/personalrobotics/feeding_web_interface/tree/main/feedingwebapp/src/ros/ros_helpers.jsx). We followed the following guiding principles when making it: - No other piece of code should need to import `roslib`; all functions that use ROSLIB should be in `ros_helpers.jsx`. - No other piece of code should need to import `react-ros`. The only exception to that is [`App.jsx`](https://github.com/personalrobotics/feeding_web_interface/tree/main/feedingwebapp/src/App.jsx) which needs to wrap elements that use ROS in the ROS tag. -- Only `connectToROS()` should call `useROS()`, while other functions should have the `ros` object passed in. This is because hooks can only be used in React components (e.g., not callback functions). So the general idea is that a component that uses ROS will call `let { isConnected, ros } = connectToROS()` within the main code for the component, and will pass `ros` to any subsequent `ros_helpers.jsx` function calls it makes. +- The best way to connect to ros would be to add the line, `const ros = useRef(useRos().ros)` to any component that needs it. Then, the functions that need `ros` object can have it passed in. This is because hooks can only be used in React components (e.g., not callback functions). So the general idea is that you will have a `ros` object defined and will be able to pass it into any subsequent `ros_helpers.jsx` function calls it makes. Note that you probably will need to import `useRef` for this to work successfully. `ros_helpers.jsx` currently supports subscribing to and publishing from ROS topics, calling ROS services, and calling ROS actions. Sample code for each of these features can be found in [`TestROS.jsx`](https://github.com/personalrobotics/feeding_web_interface/tree/main/feedingwebapp/src/ros/TestROS.jsx) and the components it imports, and can be accessed by starting the web app and navigating to `http://localhost:3000/test_ros` in your browser. See [README.md](https://github.com/personalrobotics/feeding_web_interface/tree/main/feedingwebapp/README.md) for more detailed instructions on using the "Test ROS" interface. diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js index 7afae5b7..38821625 100644 --- a/feedingwebapp/src/Pages/Constants.js +++ b/feedingwebapp/src/Pages/Constants.js @@ -44,6 +44,8 @@ export const CAMERA_FEED_TOPIC = '/camera/color/image_raw' export const FACE_DETECTION_TOPIC = '/face_detection' export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection' export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img' +// Name of the ROS topic and the type of response +export const FOOD_ON_FORK_TOPIC = {name: '/food_on_fork', type: 'std_msgs/Float32'} /** * For states that call ROS actions, this dictionary contains @@ -104,3 +106,8 @@ export const ROS_ACTION_STATUS_CANCEL_GOAL = '2' export const ROS_ACTION_STATUS_SUCCEED = '3' export const ROS_ACTION_STATUS_ABORT = '4' export const ROS_ACTION_STATUS_CANCELED = '5' + +/** + * Constant range of probability values that defines Food on Fork + */ +export const FOOD_ON_FORK_PROB_RANGE = {lowerProb: 0.4, higherProb: 0.6} diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 0004b3f4..215c65f0 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import Button from 'react-bootstrap/Button' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -7,7 +7,10 @@ import { View } from 'react-native' // Local Imports import '../Home.css' import { useGlobalState, MEAL_STATE } from '../../GlobalState' -import { MOVING_STATE_ICON_DICT } from '../../Constants' +import { FOOD_ON_FORK_TOPIC, FOOD_ON_FORK_PROB_RANGE, MOVING_STATE_ICON_DICT } from '../../Constants' + +// Import subscriber to be able to subscribe to FoF topic +import { subscribeToROSTopic, unsubscribeFromROSTopic, useROS } from '../../../ros/ros_helpers' /** * The BiteDone component appears after the robot has moved to the user's mouth, @@ -32,6 +35,27 @@ const BiteDone = () => { let iconWidth = isPortrait ? '28vh' : '28vw' let iconHeight = isPortrait ? '18vh' : '18vw' + const [foodProb, setFoodProb] = useState([]) + + // Connect to Ros + const ros = useRef(useROS().ros) + + useEffect(() => { + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, (message) => setFoodProb(prevVal => [...prevVal, Number(message.data)]), 1000) + + return () => { + if (foodProb.length > 10 && foodProb[9] < FOOD_ON_FORK_PROB_RANGE.lowerProb) { + console.log("prob in range") + unsubscribeFromROSTopic(food_on_fork_topic, () => { + console.log("Unsubscribed from FoF!") + }) + moveAbovePlate() + } + } + }) + + + /** * Callback function for when the user wants to move above plate. */ diff --git a/feedingwebapp/src/ros/ros_helpers.js b/feedingwebapp/src/ros/ros_helpers.js index f94b200a..9d39b960 100644 --- a/feedingwebapp/src/ros/ros_helpers.js +++ b/feedingwebapp/src/ros/ros_helpers.js @@ -55,9 +55,14 @@ export function createROSTopic(ros, topicName, topicType) { * @param {string} topicType The type of the topic to create. * @param {function} callback The callback function to call when a message is * received. + * @param {number} interval The interval value in ms (milliseconds) that the + * topic should be listened at (for instance, 1000 would mean + * that we subscribe to the particular topic and listen to it + * every 1000 ms). Default value is 0; which means as soon as + * the message is received * @returns {object} The ROSLIB.Topic, or null if ROS is not connected. */ -export function subscribeToROSTopic(ros, topicName, topicType, callback) { +export function subscribeToROSTopic(ros, topicName, topicType, callback, interval=0) { if (ros === null) { console.log('ROS is not connected') return null @@ -65,7 +70,8 @@ export function subscribeToROSTopic(ros, topicName, topicType, callback) { let topic = new ROSLIB.Topic({ ros: ros, name: topicName, - messageType: topicType + messageType: topicType, + throttle_rate: interval }) topic.subscribe(callback) return topic From 1907c7a3e070be7b42020950576a75e55ea394f3 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Thu, 3 Aug 2023 18:52:04 -0700 Subject: [PATCH 03/10] formatted --- feedingwebapp/src/Pages/Constants.js | 4 ++-- .../src/Pages/Home/MealStates/BiteDone.jsx | 14 +++++++++----- feedingwebapp/src/ros/ros_helpers.js | 10 +++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js index 38821625..f657eec3 100644 --- a/feedingwebapp/src/Pages/Constants.js +++ b/feedingwebapp/src/Pages/Constants.js @@ -45,7 +45,7 @@ export const FACE_DETECTION_TOPIC = '/face_detection' export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection' export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img' // Name of the ROS topic and the type of response -export const FOOD_ON_FORK_TOPIC = {name: '/food_on_fork', type: 'std_msgs/Float32'} +export const FOOD_ON_FORK_TOPIC = { name: '/food_on_fork', type: 'std_msgs/Float32' } /** * For states that call ROS actions, this dictionary contains @@ -110,4 +110,4 @@ export const ROS_ACTION_STATUS_CANCELED = '5' /** * Constant range of probability values that defines Food on Fork */ -export const FOOD_ON_FORK_PROB_RANGE = {lowerProb: 0.4, higherProb: 0.6} +export const FOOD_ON_FORK_PROB_RANGE = { lowerProb: 0.4, higherProb: 0.6 } diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 215c65f0..0f0e1002 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -41,21 +41,25 @@ const BiteDone = () => { const ros = useRef(useROS().ros) useEffect(() => { - const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, (message) => setFoodProb(prevVal => [...prevVal, Number(message.data)]), 1000) + const food_on_fork_topic = subscribeToROSTopic( + ros.current, + FOOD_ON_FORK_TOPIC.name, + FOOD_ON_FORK_TOPIC.type, + (message) => setFoodProb((prevVal) => [...prevVal, Number(message.data)]), + 1000 + ) return () => { if (foodProb.length > 10 && foodProb[9] < FOOD_ON_FORK_PROB_RANGE.lowerProb) { - console.log("prob in range") + console.log('prob in range') unsubscribeFromROSTopic(food_on_fork_topic, () => { - console.log("Unsubscribed from FoF!") + console.log('Unsubscribed from FoF!') }) moveAbovePlate() } } }) - - /** * Callback function for when the user wants to move above plate. */ diff --git a/feedingwebapp/src/ros/ros_helpers.js b/feedingwebapp/src/ros/ros_helpers.js index 9d39b960..ffeb6e69 100644 --- a/feedingwebapp/src/ros/ros_helpers.js +++ b/feedingwebapp/src/ros/ros_helpers.js @@ -55,14 +55,14 @@ export function createROSTopic(ros, topicName, topicType) { * @param {string} topicType The type of the topic to create. * @param {function} callback The callback function to call when a message is * received. - * @param {number} interval The interval value in ms (milliseconds) that the + * @param {number} interval The interval value in ms (milliseconds) that the * topic should be listened at (for instance, 1000 would mean - * that we subscribe to the particular topic and listen to it - * every 1000 ms). Default value is 0; which means as soon as + * that we subscribe to the particular topic and listen to it + * every 1000 ms). Default value is 0; which means as soon as * the message is received * @returns {object} The ROSLIB.Topic, or null if ROS is not connected. */ -export function subscribeToROSTopic(ros, topicName, topicType, callback, interval=0) { +export function subscribeToROSTopic(ros, topicName, topicType, callback, interval = 0) { if (ros === null) { console.log('ROS is not connected') return null @@ -70,7 +70,7 @@ export function subscribeToROSTopic(ros, topicName, topicType, callback, interva let topic = new ROSLIB.Topic({ ros: ros, name: topicName, - messageType: topicType, + messageType: topicType, throttle_rate: interval }) topic.subscribe(callback) From df980006822f65148aef1110b46518d418ed3b79 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Fri, 4 Aug 2023 12:23:45 -0700 Subject: [PATCH 04/10] [IP] Bite Transfer FoF integrated (with immediate motion) --- .../src/Pages/Home/MealStates/BiteDone.jsx | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 0f0e1002..390a9945 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback, useEffect, useRef, useState } from 'react' +import React, { useCallback, useEffect, useRef } from 'react' import Button from 'react-bootstrap/Button' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -35,30 +35,26 @@ const BiteDone = () => { let iconWidth = isPortrait ? '28vh' : '28vw' let iconHeight = isPortrait ? '18vh' : '18vw' - const [foodProb, setFoodProb] = useState([]) - // Connect to Ros const ros = useRef(useROS().ros) + const food_on_fork_callback = (message) => { + // setFoodProb((prevVal) => [...prevVal, Number(message.data)]) + if (Number(message.data) < FOOD_ON_FORK_PROB_RANGE.lowerProb) { + console.log('moving above plate') + moveAbovePlate() + return + } + } + useEffect(() => { - const food_on_fork_topic = subscribeToROSTopic( - ros.current, - FOOD_ON_FORK_TOPIC.name, - FOOD_ON_FORK_TOPIC.type, - (message) => setFoodProb((prevVal) => [...prevVal, Number(message.data)]), - 1000 - ) + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) return () => { - if (foodProb.length > 10 && foodProb[9] < FOOD_ON_FORK_PROB_RANGE.lowerProb) { - console.log('prob in range') - unsubscribeFromROSTopic(food_on_fork_topic, () => { - console.log('Unsubscribed from FoF!') - }) - moveAbovePlate() - } + console.log('unscubscribed from FoF') + unsubscribeFromROSTopic(food_on_fork_topic) } - }) + }, [setMealState]) /** * Callback function for when the user wants to move above plate. From 0b11338684945b2fbe97dc34a99820e2ca69c193 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Fri, 4 Aug 2023 12:31:28 -0700 Subject: [PATCH 05/10] [IP] minor change --- feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 390a9945..f80e1f95 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -54,7 +54,7 @@ const BiteDone = () => { console.log('unscubscribed from FoF') unsubscribeFromROSTopic(food_on_fork_topic) } - }, [setMealState]) + }, [setMealState, food_on_fork_callback]) /** * Callback function for when the user wants to move above plate. From 5d6ea8732c1b4cec0b87f782ad0100281f1954f2 Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Fri, 4 Aug 2023 16:11:01 -0700 Subject: [PATCH 06/10] [IP] added a call back function --- feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index f80e1f95..311a7573 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -38,14 +38,15 @@ const BiteDone = () => { // Connect to Ros const ros = useRef(useROS().ros) - const food_on_fork_callback = (message) => { + const food_on_fork_callback = useCallback((message) => { // setFoodProb((prevVal) => [...prevVal, Number(message.data)]) + console.log("Prob: " + message.data); if (Number(message.data) < FOOD_ON_FORK_PROB_RANGE.lowerProb) { console.log('moving above plate') moveAbovePlate() return } - } + }) useEffect(() => { const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) From 3efbd45dc30ec642ccd4d23bc0f15878386e3aaf Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Sun, 24 Sep 2023 17:01:20 -0700 Subject: [PATCH 07/10] added window --- feedingwebapp/src/Pages/Constants.js | 5 ++-- feedingwebapp/src/Pages/GlobalState.jsx | 10 +++++-- .../src/Pages/Home/MealStates/BiteDone.jsx | 29 +++++++++++++------ feedingwebapp/src/Pages/Settings/Settings.jsx | 9 ++++++ package-lock.json | 6 ++++ 5 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 package-lock.json diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js index e3973b4a..97015204 100644 --- a/feedingwebapp/src/Pages/Constants.js +++ b/feedingwebapp/src/Pages/Constants.js @@ -118,6 +118,7 @@ export const ROS_ACTION_STATUS_ABORT = '4' export const ROS_ACTION_STATUS_CANCELED = '5' /** - * Constant range of probability values that defines Food on Fork + * Constant range of probability values that defines Food on Fork and window size */ -export const FOOD_ON_FORK_PROB_RANGE = { lowerProb: 0.4, higherProb: 0.6 } +export const FOOD_ON_FORK_PROB_RANGE = { lowerProb: 0.5, higherProb: 0.5 } +export const FOOD_ON_FORK_WINDOW_SIZE = 200 diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx index d90e6396..281b68df 100644 --- a/feedingwebapp/src/Pages/GlobalState.jsx +++ b/feedingwebapp/src/Pages/GlobalState.jsx @@ -83,7 +83,8 @@ export const MEAL_STATE = { export const SETTINGS = { stagingPosition: ['In Front of Me', 'On My Right Side'], biteInitiation: ['Open Mouth', 'Say "I am Ready"', 'Press Button'], - biteSelection: ['Name of Food', 'Click on Food'] + biteSelection: ['Name of Food', 'Click on Food'], + foodOnFork: ['Yes', 'No'] } /** @@ -110,6 +111,7 @@ export const useGlobalState = create( stagingPosition: SETTINGS.stagingPosition[0], biteInitiation: SETTINGS.biteInitiation[0], biteSelection: SETTINGS.biteSelection[0], + foodOnFork: SETTINGS.foodOnFork[0], // Setters for global state setMealState: (mealState) => @@ -144,7 +146,11 @@ export const useGlobalState = create( setBiteSelection: (biteSelection) => set(() => ({ biteSelection: biteSelection - })) + })), + setFoodOnFork: (foodOnFork) => + set(() => ({ + foodOnFork: foodOnFork + })), }), { name: 'ada_web_app_global_state' } ) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 7fb4f4a6..36b0d82b 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -7,7 +7,7 @@ import { View } from 'react-native' // Local Imports import '../Home.css' import { useGlobalState, MEAL_STATE } from '../../GlobalState' -import { FOOD_ON_FORK_TOPIC, FOOD_ON_FORK_PROB_RANGE, MOVING_STATE_ICON_DICT } from '../../Constants' +import { FOOD_ON_FORK_TOPIC, FOOD_ON_FORK_PROB_RANGE, FOOD_ON_FORK_WINDOW_SIZE, MOVING_STATE_ICON_DICT } from '../../Constants' // Import subscriber to be able to subscribe to FoF topic import { subscribeToROSTopic, unsubscribeFromROSTopic, useROS } from '../../../ros/ros_helpers' @@ -20,6 +20,7 @@ import { subscribeToROSTopic, unsubscribeFromROSTopic, useROS } from '../../../r const BiteDone = () => { // Get the relevant global variables const setMealState = useGlobalState((state) => state.setMealState) + const foodOnFork = useGlobalState((state) => state.foodOnFork) // Get icon image for move above plate let moveAbovePlateImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingFromMouthToAbovePlate] // Get icon image for move to resting position @@ -37,11 +38,19 @@ const BiteDone = () => { // Connect to Ros const ros = useRef(useROS().ros) - + let window = [] const food_on_fork_callback = useCallback((message) => { - // setFoodProb((prevVal) => [...prevVal, Number(message.data)]) - console.log("Prob: " + message.data); - if (Number(message.data) < FOOD_ON_FORK_PROB_RANGE.lowerProb) { + if (window.size == FOOD_ON_FORK_WINDOW_SIZE) { + window.shift() + } + window.push(Number(message.data)) + let countLessThanRange = 0 + for (const val of window) { + if (val < FOOD_ON_FORK_PROB_RANGE.lowerProb) { + countLessThanRange++ + } + } + if (window.size == FOOD_ON_FORK_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_WINDOW_SIZE) { console.log('moving above plate') moveAbovePlate() return @@ -49,11 +58,13 @@ const BiteDone = () => { }) useEffect(() => { - const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) + if (foodOnFork == "Yes") { + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) - return () => { - console.log('unscubscribed from FoF') - unsubscribeFromROSTopic(food_on_fork_topic) + return () => { + console.log('unscubscribed from FoF') + unsubscribeFromROSTopic(food_on_fork_topic) + } } }, [setMealState, food_on_fork_callback]) diff --git a/feedingwebapp/src/Pages/Settings/Settings.jsx b/feedingwebapp/src/Pages/Settings/Settings.jsx index b0865e87..ac76078f 100644 --- a/feedingwebapp/src/Pages/Settings/Settings.jsx +++ b/feedingwebapp/src/Pages/Settings/Settings.jsx @@ -60,6 +60,15 @@ const Settings = () => { valueSetter={useGlobalState((state) => state.setBiteSelection)} /> + + + Would you like to use the Automatic Food on Fork detection? + state.foodOnFork)} + valueSetter={useGlobalState((state) => state.setFoodOnFork)} + /> + ) } diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..32301a86 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "feeding_web_interface", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} From 213f8f3a2e30ac905cf2d96e58d90942bc10f6ec Mon Sep 17 00:00:00 2001 From: Atharva Kashyap Date: Sun, 24 Sep 2023 17:08:16 -0700 Subject: [PATCH 08/10] added window for FoF in in-front of face --- feedingwebapp/src/Pages/GlobalState.jsx | 2 +- feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx index 281b68df..0fc29d50 100644 --- a/feedingwebapp/src/Pages/GlobalState.jsx +++ b/feedingwebapp/src/Pages/GlobalState.jsx @@ -150,7 +150,7 @@ export const useGlobalState = create( setFoodOnFork: (foodOnFork) => set(() => ({ foodOnFork: foodOnFork - })), + })) }), { name: 'ada_web_app_global_state' } ) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 36b0d82b..f3956152 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -40,7 +40,7 @@ const BiteDone = () => { const ros = useRef(useROS().ros) let window = [] const food_on_fork_callback = useCallback((message) => { - if (window.size == FOOD_ON_FORK_WINDOW_SIZE) { + if (window.size === FOOD_ON_FORK_WINDOW_SIZE) { window.shift() } window.push(Number(message.data)) @@ -50,7 +50,7 @@ const BiteDone = () => { countLessThanRange++ } } - if (window.size == FOOD_ON_FORK_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_WINDOW_SIZE) { + if (window.size === FOOD_ON_FORK_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_WINDOW_SIZE) { console.log('moving above plate') moveAbovePlate() return @@ -58,7 +58,7 @@ const BiteDone = () => { }) useEffect(() => { - if (foodOnFork == "Yes") { + if (foodOnFork === 'Yes') { const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) return () => { @@ -66,7 +66,7 @@ const BiteDone = () => { unsubscribeFromROSTopic(food_on_fork_topic) } } - }, [setMealState, food_on_fork_callback]) + }, [setMealState, food_on_fork_callback, foodOnFork]) /** * Callback function for when the user wants to move above plate. From 8a77faa076638d2b51262cd200728e098d51c15f Mon Sep 17 00:00:00 2001 From: atharva-kashyap Date: Wed, 27 Sep 2023 16:26:22 -0700 Subject: [PATCH 09/10] FoF has been added everywhere --- feedingwebapp/src/Pages/Constants.js | 3 +- .../Home/MealStates/BiteAcquisitionCheck.jsx | 51 ++++++++++++++++++- .../src/Pages/Home/MealStates/BiteDone.jsx | 15 ++++-- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js index 97015204..62410610 100644 --- a/feedingwebapp/src/Pages/Constants.js +++ b/feedingwebapp/src/Pages/Constants.js @@ -121,4 +121,5 @@ export const ROS_ACTION_STATUS_CANCELED = '5' * Constant range of probability values that defines Food on Fork and window size */ export const FOOD_ON_FORK_PROB_RANGE = { lowerProb: 0.5, higherProb: 0.5 } -export const FOOD_ON_FORK_WINDOW_SIZE = 200 +export const FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE = 120 +export const FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE = 50 diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx index 75b7a38f..12079aa6 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback } from 'react' +import React, { useCallback, useEffect, useRef } from 'react' import Button from 'react-bootstrap/Button' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -7,7 +7,15 @@ import { View } from 'react-native' // Local Imports import '../Home.css' import { useGlobalState, MEAL_STATE } from '../../GlobalState' -import { MOVING_STATE_ICON_DICT } from '../../Constants' +import { + FOOD_ON_FORK_TOPIC, + FOOD_ON_FORK_PROB_RANGE, + FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE, + MOVING_STATE_ICON_DICT +} from '../../Constants' + +// Import subscriber to be able to subscribe to FoF topic +import { subscribeToROSTopic, unsubscribeFromROSTopic, useROS } from '../../../ros/ros_helpers' /** * The BiteAcquisitionCheck component appears after the robot has attempted to @@ -16,6 +24,7 @@ import { MOVING_STATE_ICON_DICT } from '../../Constants' const BiteAcquisitionCheck = () => { // Get the relevant global variables const setMealState = useGlobalState((state) => state.setMealState) + const foodOnFork = useGlobalState((state) => state.foodOnFork) // Get icon image for move above plate let moveAbovePlateImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingAbovePlate] // Get icon image for move to mouth @@ -31,6 +40,44 @@ const BiteAcquisitionCheck = () => { let iconWidth = isPortrait ? '28vh' : '28vw' let iconHeight = isPortrait ? '18vh' : '18vw' + // Connect to Ros + const ros = useRef(useROS().ros) + let window = [] + const food_on_fork_callback = useCallback((message) => { + console.log('Subscribed to FoF') + if (window.length === Number(FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE)) { + console.log('entered') + window.shift() + } + window.push(Number(message.data)) + let countLessThanRange = 0 + for (const val of window) { + if (val < FOOD_ON_FORK_PROB_RANGE.lowerProb) { + countLessThanRange++ + } + } + console.log(window.length) + if ( + window.length === FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE && + countLessThanRange >= 0.75 * FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE + ) { + console.log('Detecting no food on fork (Acquisition Failure); moving above plate') + acquisitionFailure() + return + } + }) + + useEffect(() => { + if (foodOnFork === 'Yes') { + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) + + return () => { + console.log('unscubscribed from FoF') + unsubscribeFromROSTopic(food_on_fork_topic) + } + } + }, [setMealState, food_on_fork_callback, foodOnFork]) + /** * Callback function for when the user indicates that the bite acquisition * succeeded. diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index f3956152..87909333 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -7,7 +7,12 @@ import { View } from 'react-native' // Local Imports import '../Home.css' import { useGlobalState, MEAL_STATE } from '../../GlobalState' -import { FOOD_ON_FORK_TOPIC, FOOD_ON_FORK_PROB_RANGE, FOOD_ON_FORK_WINDOW_SIZE, MOVING_STATE_ICON_DICT } from '../../Constants' +import { + FOOD_ON_FORK_TOPIC, + FOOD_ON_FORK_PROB_RANGE, + FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE, + MOVING_STATE_ICON_DICT +} from '../../Constants' // Import subscriber to be able to subscribe to FoF topic import { subscribeToROSTopic, unsubscribeFromROSTopic, useROS } from '../../../ros/ros_helpers' @@ -40,7 +45,8 @@ const BiteDone = () => { const ros = useRef(useROS().ros) let window = [] const food_on_fork_callback = useCallback((message) => { - if (window.size === FOOD_ON_FORK_WINDOW_SIZE) { + console.log('Inside FoF Callback') + if (window.length === Number(FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE)) { window.shift() } window.push(Number(message.data)) @@ -50,8 +56,9 @@ const BiteDone = () => { countLessThanRange++ } } - if (window.size === FOOD_ON_FORK_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_WINDOW_SIZE) { - console.log('moving above plate') + console.log(window.length) + if (window.length === FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE) { + console.log('Detecting no food on fork; moving above plate') moveAbovePlate() return } From 6ce30bac305839ce3e9e73ad234e3f74c80b5840 Mon Sep 17 00:00:00 2001 From: atharva-kashyap Date: Tue, 7 Nov 2023 12:34:53 -0800 Subject: [PATCH 10/10] made minor changes --- feedingwebapp/src/Pages/GlobalState.jsx | 1 + .../Home/MealStates/BiteAcquisitionCheck.jsx | 26 +++++++++++-------- .../src/Pages/Home/MealStates/BiteDone.jsx | 18 ++++++------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx index 0fc29d50..a7768693 100644 --- a/feedingwebapp/src/Pages/GlobalState.jsx +++ b/feedingwebapp/src/Pages/GlobalState.jsx @@ -76,6 +76,7 @@ export const MEAL_STATE = { * enable multiple buttons if they so desire. * - biteSelection: Options for how the user wants to tell the robot what food * item they want next. + * - foodOnFork: Options for user to toggle FoF detection on or off * * TODO (amaln): When we connect this to ROS, each of these settings types and * value options will have to have corresponding rosparam names and value options. diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx index 12079aa6..44a1f43d 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteAcquisitionCheck.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback, useEffect, useRef } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import Button from 'react-bootstrap/Button' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -25,6 +25,7 @@ const BiteAcquisitionCheck = () => { // Get the relevant global variables const setMealState = useGlobalState((state) => state.setMealState) const foodOnFork = useGlobalState((state) => state.foodOnFork) + const [detectedFood, setDetectedFood] = useState("") // Get icon image for move above plate let moveAbovePlateImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingAbovePlate] // Get icon image for move to mouth @@ -44,7 +45,6 @@ const BiteAcquisitionCheck = () => { const ros = useRef(useROS().ros) let window = [] const food_on_fork_callback = useCallback((message) => { - console.log('Subscribed to FoF') if (window.length === Number(FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE)) { console.log('entered') window.shift() @@ -61,20 +61,23 @@ const BiteAcquisitionCheck = () => { window.length === FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_BITE_ACQUISITION_WINDOW_SIZE ) { - console.log('Detecting no food on fork (Acquisition Failure); moving above plate') - acquisitionFailure() + if (foodOnFork === "Yes") { + console.log('Detecting no food on fork (Acquisition Failure); moving above plate') + acquisitionFailure() + } else { + setDetectedFood("detected no food") + } return - } + } }) useEffect(() => { - if (foodOnFork === 'Yes') { - const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) + console.log('Subscribed to FoF') + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) - return () => { - console.log('unscubscribed from FoF') - unsubscribeFromROSTopic(food_on_fork_topic) - } + return () => { + console.log('unscubscribed from FoF') + unsubscribeFromROSTopic(food_on_fork_topic) } }, [setMealState, food_on_fork_callback, foodOnFork]) @@ -187,6 +190,7 @@ const BiteAcquisitionCheck = () => { {reacquireBiteText()} {reacquireBiteButton()} +

Currently detecting: {detectedFood}

) }, [dimension, reacquireBiteButton, reacquireBiteText, readyForBiteButton, readyForBiteText]) diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx index 87909333..2a115b6d 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteDone.jsx @@ -45,7 +45,7 @@ const BiteDone = () => { const ros = useRef(useROS().ros) let window = [] const food_on_fork_callback = useCallback((message) => { - console.log('Inside FoF Callback') + console.log('Subscribed to FoF') if (window.length === Number(FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE)) { window.shift() } @@ -58,20 +58,20 @@ const BiteDone = () => { } console.log(window.length) if (window.length === FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE && countLessThanRange >= 0.75 * FOOD_ON_FORK_BITE_TRANSFER_WINDOW_SIZE) { - console.log('Detecting no food on fork; moving above plate') - moveAbovePlate() + if (foodOnFork === "Yes") { + console.log('Detecting no food on fork; moving above plate') + moveAbovePlate() + } return } }) useEffect(() => { - if (foodOnFork === 'Yes') { - const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) + const food_on_fork_topic = subscribeToROSTopic(ros.current, FOOD_ON_FORK_TOPIC.name, FOOD_ON_FORK_TOPIC.type, food_on_fork_callback) - return () => { - console.log('unscubscribed from FoF') - unsubscribeFromROSTopic(food_on_fork_topic) - } + return () => { + console.log('unscubscribed from FoF') + unsubscribeFromROSTopic(food_on_fork_topic) } }, [setMealState, food_on_fork_callback, foodOnFork])