From 92cbdf9d33e4e2b70c46e43561da74eed8a2fef3 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 18 Nov 2024 04:25:59 +0100 Subject: [PATCH] Fix small example issues --- .gitignore | 1 + examples/piper/{assets => }/piper_left.urdf | 40 +++++++-------- .../piper/{assets => }/piper_left_pred.urdf | 40 +++++++-------- examples/piper/{assets => }/piper_right.urdf | 40 +++++++-------- .../piper/{assets => }/piper_right_pred.urdf | 40 +++++++-------- examples/piper/play_dummy_inference.yml | 17 +++++-- examples/speech-to-text/dataflow.yml | 1 + examples/translation/dataflow_en_fr.yml | 50 +++++++++++++++++++ examples/vlm/dataflow.yml | 12 +++-- examples/vlm/dataflow_rerun.yml | 12 ++--- .../dora_distil_whisper/main.py | 5 +- node-hub/dora-opus/dora_opus/main.py | 3 +- node-hub/dora-parler/dora_parler/main.py | 2 +- node-hub/dora-qwenvl/dora_qwenvl/main.py | 11 +--- node-hub/dora-rerun/pyproject.toml | 2 +- 15 files changed, 165 insertions(+), 111 deletions(-) rename examples/piper/{assets => }/piper_left.urdf (93%) rename examples/piper/{assets => }/piper_left_pred.urdf (93%) rename examples/piper/{assets => }/piper_right.urdf (93%) rename examples/piper/{assets => }/piper_right_pred.urdf (93%) create mode 100644 examples/translation/dataflow_en_fr.yml diff --git a/.gitignore b/.gitignore index 93d3608af..a1bb9947e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ examples/**/*.txt # Remove hdf and stl files *.stl +*.STL *.hdf5 # Removing images. diff --git a/examples/piper/assets/piper_left.urdf b/examples/piper/piper_left.urdf similarity index 93% rename from examples/piper/assets/piper_left.urdf rename to examples/piper/piper_left.urdf index 99fde57d9..4e1166094 100644 --- a/examples/piper/assets/piper_left.urdf +++ b/examples/piper/piper_left.urdf @@ -23,7 +23,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -37,7 +37,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -63,7 +63,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -77,7 +77,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -121,7 +121,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -135,7 +135,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -179,7 +179,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -193,7 +193,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -237,7 +237,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -251,7 +251,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -295,7 +295,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -309,7 +309,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -353,7 +353,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -367,7 +367,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -412,7 +412,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -426,7 +426,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -471,7 +471,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -485,7 +485,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -529,7 +529,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -543,7 +543,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> diff --git a/examples/piper/assets/piper_left_pred.urdf b/examples/piper/piper_left_pred.urdf similarity index 93% rename from examples/piper/assets/piper_left_pred.urdf rename to examples/piper/piper_left_pred.urdf index a74ed6c3c..6eaa246f9 100644 --- a/examples/piper/assets/piper_left_pred.urdf +++ b/examples/piper/piper_left_pred.urdf @@ -23,7 +23,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -37,7 +37,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -63,7 +63,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -77,7 +77,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -121,7 +121,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -135,7 +135,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -179,7 +179,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -193,7 +193,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -237,7 +237,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -251,7 +251,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -295,7 +295,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -309,7 +309,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -353,7 +353,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -367,7 +367,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -412,7 +412,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -426,7 +426,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -471,7 +471,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -485,7 +485,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -529,7 +529,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -543,7 +543,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> diff --git a/examples/piper/assets/piper_right.urdf b/examples/piper/piper_right.urdf similarity index 93% rename from examples/piper/assets/piper_right.urdf rename to examples/piper/piper_right.urdf index 99fde57d9..4e1166094 100644 --- a/examples/piper/assets/piper_right.urdf +++ b/examples/piper/piper_right.urdf @@ -23,7 +23,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -37,7 +37,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -63,7 +63,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -77,7 +77,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -121,7 +121,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -135,7 +135,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -179,7 +179,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -193,7 +193,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -237,7 +237,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -251,7 +251,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -295,7 +295,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -309,7 +309,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -353,7 +353,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -367,7 +367,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -412,7 +412,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -426,7 +426,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -471,7 +471,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -485,7 +485,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -529,7 +529,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -543,7 +543,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> diff --git a/examples/piper/assets/piper_right_pred.urdf b/examples/piper/piper_right_pred.urdf similarity index 93% rename from examples/piper/assets/piper_right_pred.urdf rename to examples/piper/piper_right_pred.urdf index 2541f63f1..6adf3d491 100644 --- a/examples/piper/assets/piper_right_pred.urdf +++ b/examples/piper/piper_right_pred.urdf @@ -23,7 +23,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -37,7 +37,7 @@ rpy="0 0 0" /> + filename="base_link.STL" /> @@ -63,7 +63,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -77,7 +77,7 @@ rpy="0 0 0" /> + filename="link1.STL" /> @@ -121,7 +121,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -135,7 +135,7 @@ rpy="0 0 0" /> + filename="link2.STL" /> @@ -179,7 +179,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -193,7 +193,7 @@ rpy="0 0 0" /> + filename="link3.STL" /> @@ -237,7 +237,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -251,7 +251,7 @@ rpy="0 0 0" /> + filename="link4.STL" /> @@ -295,7 +295,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -309,7 +309,7 @@ rpy="0 0 0" /> + filename="link5.STL" /> @@ -353,7 +353,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -367,7 +367,7 @@ rpy="0 0 0" /> + filename="link6.STL" /> @@ -412,7 +412,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -426,7 +426,7 @@ rpy="0 0 0" /> + filename="link7.STL" /> @@ -471,7 +471,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -485,7 +485,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -529,7 +529,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> @@ -543,7 +543,7 @@ rpy="0 0 0" /> + filename="link8.STL" /> diff --git a/examples/piper/play_dummy_inference.yml b/examples/piper/play_dummy_inference.yml index 6169f51d5..d821837c3 100644 --- a/examples/piper/play_dummy_inference.yml +++ b/examples/piper/play_dummy_inference.yml @@ -10,13 +10,24 @@ nodes: CAN_BUS: can_left - id: rerun - build: wget + build: | + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/base_link.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link1.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link2.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link3.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link4.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link5.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link6.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link7.STL + proxychains wget -nc https://raw.githubusercontent.com/agilexrobotics/Piper_ros/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes/link8.STL + pip install -e ../../node-hub/dora-rerun + pip install git+https://github.com/rerun-io/rerun-loader-python-example-urdf.git path: dora-rerun inputs: jointstate_piper_left: piper/jointstate_left jointstate_piper_right: piper/jointstate_right env: - piper_left_urdf: assets/piper_left.urdf # Make sure to download assets/meshes from https://github.com/agilexrobotics/Piper_ros/tree/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes and put them in the assets folder - piper_right_urdf: assets/piper_right.urdf # Make sure to download assets/meshes from https://github.com/agilexrobotics/Piper_ros/tree/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes and put them in the assets folder + piper_left_urdf: piper_left.urdf # Make sure to download meshes from https://github.com/agilexrobotics/Piper_ros/tree/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes and put them in the assets folder + piper_right_urdf: piper_right.urdf # Make sure to download meshes from https://github.com/agilexrobotics/Piper_ros/tree/4f22c61f96b8fb3ef3f937b99b63edb697caadf0/src/piper_description/meshes and put them in the assets folder piper_left_transform: 0 0.2 0 piper_right_transform: 0 -0.2 0 diff --git a/examples/speech-to-text/dataflow.yml b/examples/speech-to-text/dataflow.yml index 3176bd22b..bad75e7ea 100644 --- a/examples/speech-to-text/dataflow.yml +++ b/examples/speech-to-text/dataflow.yml @@ -22,6 +22,7 @@ nodes: - text env: TARGET_LANGUAGE: english + # For China # USE_MODELSCOPE_HUB: true - id: dora-rerun diff --git a/examples/translation/dataflow_en_fr.yml b/examples/translation/dataflow_en_fr.yml new file mode 100644 index 000000000..0c1747752 --- /dev/null +++ b/examples/translation/dataflow_en_fr.yml @@ -0,0 +1,50 @@ +nodes: + - id: dora-microphone + build: pip install -e ../../node-hub/dora-microphone + path: dora-microphone + outputs: + - audio + + - id: dora-vad + build: pip install -e ../../node-hub/dora-vad + path: dora-vad + inputs: + audio: dora-microphone/audio + outputs: + - audio + + - id: dora-distil-whisper + build: pip install -e ../../node-hub/dora-distil-whisper + path: dora-distil-whisper + inputs: + input: dora-vad/audio + outputs: + - text + env: + TARGET_LANGUAGE: english + # For China (Whisper Turbo, might not be available yet) + # USE_MODELSCOPE_HUB: true + HF_ENDPOINT: https://hf-mirror.com + + - id: dora-opus + build: pip install -e ../../node-hub/dora-opus + path: dora-opus + inputs: + text: dora-distil-whisper/text + outputs: + - text + env: + SOURCE_LANGUAGE: en + TARGET_LANGUAGE: fr + # For China + HF_ENDPOINT: https://hf-mirror.com + + - id: dora-rerun + build: cargo build -p dora-rerun --release + path: dora-rerun + inputs: + translated_text: dora-opus/text + original_text: dora-distil-whisper/text + env: + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 diff --git a/examples/vlm/dataflow.yml b/examples/vlm/dataflow.yml index bfb61821b..51908b378 100644 --- a/examples/vlm/dataflow.yml +++ b/examples/vlm/dataflow.yml @@ -1,9 +1,9 @@ nodes: - id: camera - build: pip install ../../node-hub/opencv-video-capture + build: pip install -e ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: dora/timer/millis/20 + tick: dora/timer/millis/50 outputs: - image env: @@ -18,15 +18,17 @@ nodes: image: source: camera/image queue_size: 1 - tick: dora/timer/millis/100 + tick: dora/timer/millis/400 outputs: - text - tick env: - DEFAULT_QUESTION: Describe the image. + DEFAULT_QUESTION: Describe the image in a very short sentence. + # For China + # USE_MODELSCOPE_HUB: true - id: plot - build: pip install ../../node-hub/opencv-plot + build: pip install -e ../../node-hub/opencv-plot path: opencv-plot inputs: image: diff --git a/examples/vlm/dataflow_rerun.yml b/examples/vlm/dataflow_rerun.yml index 785779e23..6c933a2f3 100644 --- a/examples/vlm/dataflow_rerun.yml +++ b/examples/vlm/dataflow_rerun.yml @@ -3,7 +3,7 @@ nodes: build: pip install -e ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: dora/timer/millis/20 + tick: dora/timer/millis/50 outputs: - image env: @@ -18,12 +18,13 @@ nodes: image: source: camera/image queue_size: 1 - tick: dora/timer/millis/200 + tick: dora/timer/millis/400 outputs: - text - tick env: - DEFAULT_QUESTION: Describe the image. + DEFAULT_QUESTION: Describe the image in a very short sentence. + # USE_MODELSCOPE_HUB: true - id: plot build: cargo build -p dora-rerun --release @@ -37,7 +38,4 @@ nodes: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 README: | - # Visualization of your webcam - - - You can increase or decrease the frequency within the dataflow with: `tick: dora/timer/millis/20` in the webcam node. - - You can change the model prompt within the dataflow with: `DEFAULT_QUESTION: Describe the image.` + # Visualization of QwenVL2 diff --git a/node-hub/dora-distil-whisper/dora_distil_whisper/main.py b/node-hub/dora-distil-whisper/dora_distil_whisper/main.py index 7f058ad3c..2bc27861d 100644 --- a/node-hub/dora-distil-whisper/dora_distil_whisper/main.py +++ b/node-hub/dora-distil-whisper/dora_distil_whisper/main.py @@ -7,12 +7,12 @@ DEFAULT_PATH = "openai/whisper-large-v3-turbo" TARGET_LANGUAGE = os.getenv("TARGET_LANGUAGE", "chinese") -TRANSLATE = bool(os.getenv("TRANSLATE", "False")) +TRANSLATE = bool(os.getenv("TRANSLATE", "False") in ["True", "true"]) MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", DEFAULT_PATH) -if bool(os.getenv("USE_MODELSCOPE_HUB")) is True: +if bool(os.getenv("USE_MODELSCOPE_HUB") in ["True", "true"]): from modelscope import snapshot_download if not Path(MODEL_NAME_OR_PATH).exists(): @@ -107,5 +107,4 @@ def main(): if result["text"] in BAD_SENTENCES: continue text = cut_repetition(result["text"]) - print(text, flush=True) node.send_output("text", pa.array([text]), {"language": TARGET_LANGUAGE}) diff --git a/node-hub/dora-opus/dora_opus/main.py b/node-hub/dora-opus/dora_opus/main.py index e73d4ebbd..b0c6305a5 100644 --- a/node-hub/dora-opus/dora_opus/main.py +++ b/node-hub/dora-opus/dora_opus/main.py @@ -5,6 +5,7 @@ import numpy as np from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + from_code = os.getenv("SOURCE_LANGUAGE", "zh") to_code = os.getenv("TARGET_LANGUAGE", "en") DEFAULT_PATH = f"Helsinki-NLP/opus-mt-{from_code}-{to_code}" @@ -12,7 +13,7 @@ MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", DEFAULT_PATH) -if bool(os.getenv("USE_MODELSCOPE_HUB")) is True: +if bool(os.getenv("USE_MODELSCOPE_HUB") in ["True", "true"]): from modelscope import snapshot_download if not Path(MODEL_NAME_OR_PATH).exists(): diff --git a/node-hub/dora-parler/dora_parler/main.py b/node-hub/dora-parler/dora_parler/main.py index 6547ac421..80ee72815 100644 --- a/node-hub/dora-parler/dora_parler/main.py +++ b/node-hub/dora-parler/dora_parler/main.py @@ -24,7 +24,7 @@ MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", DEFAULT_PATH) -if bool(os.getenv("USE_MODELSCOPE_HUB")) is True: +if bool(os.getenv("USE_MODELSCOPE_HUB") in ["True", "true"]): from modelscope import snapshot_download if not Path(MODEL_NAME_OR_PATH).exists(): diff --git a/node-hub/dora-qwenvl/dora_qwenvl/main.py b/node-hub/dora-qwenvl/dora_qwenvl/main.py index a9eb26974..5dbec5cfb 100644 --- a/node-hub/dora-qwenvl/dora_qwenvl/main.py +++ b/node-hub/dora-qwenvl/dora_qwenvl/main.py @@ -12,7 +12,7 @@ MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", DEFAULT_PATH) -if bool(os.getenv("USE_MODELSCOPE_HUB")) is True: +if bool(os.getenv("USE_MODELSCOPE_HUB") in ["True", "true"]): from modelscope import snapshot_download if not Path(MODEL_NAME_OR_PATH).exists(): @@ -111,15 +111,6 @@ def main(): if event_type == "INPUT": - # pylint: disable=fixme - # TODO: Remove this after https://github.com/dora-rs/dora/pull/652 - while True: - next_event = node.next(timeout=0.001) - if next_event is not None and next_event["type"] == "INPUT": - event = next_event - else: - break - event_id = event["id"] if "image" in event_id: diff --git a/node-hub/dora-rerun/pyproject.toml b/node-hub/dora-rerun/pyproject.toml index 432362263..8e5aa0fb1 100644 --- a/node-hub/dora-rerun/pyproject.toml +++ b/node-hub/dora-rerun/pyproject.toml @@ -12,4 +12,4 @@ dependencies = [ scripts = { "dora-rerun" = "dora_rerun:py_main" } [tool.maturin] -features = ["pyo3/extension-module"] +features = ["python", "pyo3/extension-module"]