diff --git a/feedingwebapp/README.md b/feedingwebapp/README.md index 92c8f572..810b1002 100644 --- a/feedingwebapp/README.md +++ b/feedingwebapp/README.md @@ -13,6 +13,7 @@ The overall user flow for this robot can be seen below. ## Dependencies - [Node.js](https://nodejs.org/en/download/package-manager) - [`serve` must be globally installed](https://create-react-app.dev/docs/deployment/) (ideally with `sudo`): `sudo npm install -g serve` +- [`pm2` must be globally installed](https://pm2.keymetrics.io/docs/usage/quick-start/): `npm install pm2@latest -g` ## Getting Started in Computer @@ -23,7 +24,7 @@ The overall user flow for this robot can be seen below. 4. Source the directory: `source install/setup.bash` 5. Navigate to the web app folder: `cd feeding_web_interface/feedingwebapp` 6. Install web app dependencies: `npm install --legacy-peer-deps` - 1. You may also have to run `npx playwright install`; you might be prompted to run that after `node --env-file=.env start_robot_browser.js` + 1. You may also have to run `npx playwright install`; you might be prompted to run that after `node start_robot_browser.js` * Consider checking out the Troubleshooting section if there are errors in this process. If your workspace has already been built, you should run `source install/setup.bash`. If this is your first time building your workspace, you should `source /opt/ros/humble/setup.bash` and then run `colcon build` followed by `source install/setup.bash`. For both of the above cases, you must be in the main directory of your workspace (e.g., `src` should be a subfolder). @@ -33,7 +34,7 @@ If your workspace has already been built, you should run `source install/setup.b - If you're not running the robot code alongside the app, set `REACT_APP_DEBUG=true` in `.env` to be able to move past screens where the app is waiting on the robot. The default is `REACT_APP_DEBUG=false`. - If users will be accessing the app on a device other than the device running ROS, change `REACT_APP_ROS_SERVER_HOSTNAME` in `.env` to be the hostname of the device running ROS. Ensure that device is configured so that ports 8080 (web_video_server default) and 9090 (rosbridge default) can be accessed. 3. Start the app: `npm start` -4. Start the WebRTC signalling server: `node --env-file=.env server.js` +4. Start the WebRTC signalling server: `pm2 start server.js` (see [here](https://pm2.keymetrics.io/docs/usage/quick-start/) for `pm2`` instructions) 5. Start the headless robot browser: `node start_robot_browser.js` 6. Use a web browser to navigate to `localhost:3000` to see the application. diff --git a/feedingwebapp/package-lock.json b/feedingwebapp/package-lock.json index c09eed55..4cde9e55 100644 --- a/feedingwebapp/package-lock.json +++ b/feedingwebapp/package-lock.json @@ -18,6 +18,7 @@ "bootstrap": "^5.1.3", "caniuse-lite": "^1.0.30001579", "cors": "^2.8.5", + "dotenv": "^16.4.1", "express": "^4.18.2", "mdb-react-ui-kit": "^3.0.0", "minimist": "^1.2.8", @@ -9240,6 +9241,17 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/dotenv": { + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.1.tgz", + "integrity": "sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/dotenv-expand": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", @@ -20106,9 +20118,9 @@ } }, "node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/tsutils": { "version": "3.21.0", diff --git a/feedingwebapp/package.json b/feedingwebapp/package.json index 378efbad..40abaa86 100644 --- a/feedingwebapp/package.json +++ b/feedingwebapp/package.json @@ -13,6 +13,7 @@ "bootstrap": "^5.1.3", "caniuse-lite": "^1.0.30001579", "cors": "^2.8.5", + "dotenv": "^16.4.1", "express": "^4.18.2", "mdb-react-ui-kit": "^3.0.0", "minimist": "^1.2.8", diff --git a/feedingwebapp/server.js b/feedingwebapp/server.js index 4cc271da..93ef1ffa 100644 --- a/feedingwebapp/server.js +++ b/feedingwebapp/server.js @@ -8,6 +8,7 @@ const SegfaultHandler = require('segfault-handler') SegfaultHandler.registerHandler('crash.log') +require('dotenv').config() const express = require('express') const app = express() const bodyParser = require('body-parser') @@ -40,86 +41,86 @@ app.post('/subscribe', async ({ body }, res) => { ] }) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: created peer connection object") + console.log(Date(Date.now()).toString(), 'subscribe: created peer connection object') } // Close any old peers on the same IP address const topic = body.topic if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: got topic", topic) + console.log(Date(Date.now()).toString(), 'subscribe: got topic', topic) } const key = body.ip + ':' + topic if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: got key", key) + console.log(Date(Date.now()).toString(), 'subscribe: got key', key) } if (key in subscribePeers && subscribePeers[key] && subscribePeers[key].connectionState !== 'closed') { if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: peer for key already exists") + console.log(Date(Date.now()).toString(), 'subscribe: peer for key already exists') } const senders = subscribePeers[key].getSenders() if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: got senders for peer for key") + console.log(Date(Date.now()).toString(), 'subscribe: got senders for peer for key') } senders.forEach((sender) => subscribePeers[key].removeTrack(sender)) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: removed tracks") + console.log(Date(Date.now()).toString(), 'subscribe: removed tracks') } subscribePeers[key].close() if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: closed peer connection") + console.log(Date(Date.now()).toString(), 'subscribe: closed peer connection') } } subscribePeers[key] = peer if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: set new peer connection") + console.log(Date(Date.now()).toString(), 'subscribe: set new peer connection') } const desc = new webrtc.RTCSessionDescription(body.sdp) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: created desc") + console.log(Date(Date.now()).toString(), 'subscribe: created desc') } await peer.setRemoteDescription(desc) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: set remote desc") + console.log(Date(Date.now()).toString(), 'subscribe: set remote desc') } // Add the publisher's video stream to the subscriber's peer connection if (topic in senderStream) { if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: adding topics from publisher") + console.log(Date(Date.now()).toString(), 'subscribe: adding topics from publisher') } senderStream[topic].getTracks().forEach((track) => peer.addTrack(track, senderStream[topic])) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: added topics from publisher") + console.log(Date(Date.now()).toString(), 'subscribe: added topics from publisher') } } // Create an answer to the publisher's offer const answer = await peer.createAnswer() if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: created answer") + console.log(Date(Date.now()).toString(), 'subscribe: created answer') } await peer.setLocalDescription(answer) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: set local description") + console.log(Date(Date.now()).toString(), 'subscribe: set local description') } const payload = { - sdp: answer + sdp: peer.localDescription } if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: created payload") + console.log(Date(Date.now()).toString(), 'subscribe: created payload') } // Send the answer to the publisher res.json(payload) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: set payload") + console.log(Date(Date.now()).toString(), 'subscribe: set payload') } } catch (err) { console.error(Date(Date.now()).toString(), 'subscribe: Failed to process subscriber, exception: ' + err.message) res.sendStatus(500) if (debug) { - console.log(Date(Date.now()).toString(), "subscribe: sent error status 500") + console.log(Date(Date.now()).toString(), 'subscribe: sent error status 500') } } }) @@ -137,80 +138,80 @@ app.post('/publish', async ({ body }, res) => { ] }) if (debug) { - console.log(Date(Date.now()).toString(), "publish: create peer") + console.log(Date(Date.now()).toString(), 'publish: create peer') } // Close any old peers on the same IP address const topic = body.topic if (debug) { - console.log(Date(Date.now()).toString(), "publish: got topic", topic) + console.log(Date(Date.now()).toString(), 'publish: got topic', topic) } const key = body.ip + ':' + topic if (debug) { - console.log(Date(Date.now()).toString(), "publish: got key", key) + console.log(Date(Date.now()).toString(), 'publish: got key', key) } if (key in publishPeers && publishPeers[key] && publishPeers[key].connectionState !== 'closed') { if (debug) { - console.log(Date(Date.now()).toString(), "publish: found existing publisher for key", key) + console.log(Date(Date.now()).toString(), 'publish: found existing publisher for key', key) } const senders = publishPeers[key].getSenders() if (debug) { - console.log(Date(Date.now()).toString(), "publish: got senders for old key") + console.log(Date(Date.now()).toString(), 'publish: got senders for old key') } senders.forEach((sender) => publishPeers[key].removeTrack(sender)) if (debug) { - console.log(Date(Date.now()).toString(), "publish: removed tracks for old key") + console.log(Date(Date.now()).toString(), 'publish: removed tracks for old key') } publishPeers[key].close() if (debug) { - console.log(Date(Date.now()).toString(), "publish: closed old peer connection") + console.log(Date(Date.now()).toString(), 'publish: closed old peer connection') } } publishPeers[key] = peer if (debug) { - console.log(Date(Date.now()).toString(), "publish: added new peer") + console.log(Date(Date.now()).toString(), 'publish: added new peer') } // Send the publisher's video stream to all subscribers on that topic peer.ontrack = (e) => handleTrackEvent(e, topic) if (debug) { - console.log(Date(Date.now()).toString(), "publish: handled track events") + console.log(Date(Date.now()).toString(), 'publish: handled track events') } // Create an answer to the publisher's offer const desc = new webrtc.RTCSessionDescription(body.sdp) if (debug) { - console.log(Date(Date.now()).toString(), "publish: got desc") + console.log(Date(Date.now()).toString(), 'publish: got desc') } await peer.setRemoteDescription(desc) if (debug) { - console.log(Date(Date.now()).toString(), "publish: set remote description") + console.log(Date(Date.now()).toString(), 'publish: set remote description') } const answer = await peer.createAnswer() if (debug) { - console.log(Date(Date.now()).toString(), "publish: got answer") + console.log(Date(Date.now()).toString(), 'publish: got answer') } await peer.setLocalDescription(answer) if (debug) { - console.log(Date(Date.now()).toString(), "publish: set local description") + console.log(Date(Date.now()).toString(), 'publish: set local description') } const payload = { - sdp: answer + sdp: peer.localDescription } if (debug) { - console.log(Date(Date.now()).toString(), "publish: created payload") + console.log(Date(Date.now()).toString(), 'publish: created payload') } // Send the answer to the publisher res.json(payload) if (debug) { - console.log(Date(Date.now()).toString(), "publish: set json payload") + console.log(Date(Date.now()).toString(), 'publish: set json payload') } } catch (err) { console.error(Date(Date.now()).toString(), 'publish: Failed to process publisher, exception: ' + err.message) res.sendStatus(500) if (debug) { - console.log(Date(Date.now()).toString(), "publish: sent error status 500") + console.log(Date(Date.now()).toString(), 'publish: sent error status 500') } } }) diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx index 746f1d58..36372cc3 100644 --- a/feedingwebapp/src/Pages/GlobalState.jsx +++ b/feedingwebapp/src/Pages/GlobalState.jsx @@ -128,7 +128,7 @@ export const useGlobalState = create( // Flag to indicate robot motion trough teleoperation interface teleopIsMoving: false, // Flag to indicate whether to auto-continue after face detection - faceDetectionAutoContinue: false, + faceDetectionAutoContinue: true, // Whether the settings bite transfer page is currently at the user's face // or not. This is in the off-chance that the mealState is not at the user's // face, the settings page is, and the user refreshes -- the page should diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx index 6cc8dd66..2e327b1f 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx @@ -39,9 +39,9 @@ const DetectingFace = (props) => { // Font size for text let textFontSize = 3 // let buttonWidth = 22 - let buttonHeight = 14 + let buttonHeight = 12 let iconWidth = 20 - let iconHeight = 12 + let iconHeight = 10 let sizeSuffix = isPortrait ? 'vh' : 'vw' /** diff --git a/feedingwebapp/src/buttons/MaskButton.jsx b/feedingwebapp/src/buttons/MaskButton.jsx index 77cd91bb..d667da83 100644 --- a/feedingwebapp/src/buttons/MaskButton.jsx +++ b/feedingwebapp/src/buttons/MaskButton.jsx @@ -46,16 +46,20 @@ function MaskButton(props) { const drawCircle = useCallback(() => { // Get the canvas const canvas = canvasRef.current - // Get the context - const ctx = canvas.getContext('2d') - // Clear the canvas - ctx.clearRect(0, 0, canvas.width, canvas.height) - // Draw a red filled circle - let radius = 5 - ctx.beginPath() - ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI) - ctx.fillStyle = 'red' - ctx.fill() + // Canvas might be null if the previous render had more MaskButtons + // than the current render. + if (canvas !== null) { + // Get the context + const ctx = canvas.getContext('2d') + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height) + // Draw a red filled circle + let radius = 5 + ctx.beginPath() + ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI) + ctx.fillStyle = 'red' + ctx.fill() + } }, []) // Draw a red filled circle on the middle of the canvas diff --git a/feedingwebapp/src/webrtc/webrtc_helpers.js b/feedingwebapp/src/webrtc/webrtc_helpers.js index bafae5ab..e866253f 100644 --- a/feedingwebapp/src/webrtc/webrtc_helpers.js +++ b/feedingwebapp/src/webrtc/webrtc_helpers.js @@ -43,6 +43,7 @@ export class WebRTCConnection { this.peerConnection.onnegotiationneeded = async () => { try { + console.log('onnegotiationneeded') const offer = await this.peerConnection.createOffer() await this.peerConnection.setLocalDescription(offer) const ip = await this.getIPAddress() @@ -61,6 +62,7 @@ export class WebRTCConnection { } this.peerConnection.oniceconnectionstatechange = () => { + console.log('oniceconnectionstatechange') if (!this.peerConnection) throw new Error('peerConnection is undefined') if (this.peerConnection.iceConnectionState === 'failed') { this.peerConnection.restartIce() diff --git a/feedingwebapp/start_robot_browser.js b/feedingwebapp/start_robot_browser.js index f25a1484..febb1750 100644 --- a/feedingwebapp/start_robot_browser.js +++ b/feedingwebapp/start_robot_browser.js @@ -41,6 +41,9 @@ if (argv.port) { ] }) const page = await browser.newPage() + page.on('console', (msg) => { + console.log(msg) + }) while (num_tries < max_tries) { try {