From 00e405f5c047efd43daf1177f2e18b68a193a0f4 Mon Sep 17 00:00:00 2001 From: AYokochi Date: Mon, 27 Nov 2023 15:45:54 +0900 Subject: [PATCH 1/3] add text to speech --- cube_petit_text_to_speech/CMakeLists.txt | 32 ++ .../launch/cube_petit_text_to_speech.launch | 5 + cube_petit_text_to_speech/package.xml | 52 +++ cube_petit_text_to_speech/scripts/jtalk.py | 383 ++++++++++++++++++ .../scripts/text_to_emotion.py | 92 +++++ 5 files changed, 564 insertions(+) create mode 100644 cube_petit_text_to_speech/CMakeLists.txt create mode 100644 cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch create mode 100644 cube_petit_text_to_speech/package.xml create mode 100755 cube_petit_text_to_speech/scripts/jtalk.py create mode 100755 cube_petit_text_to_speech/scripts/text_to_emotion.py diff --git a/cube_petit_text_to_speech/CMakeLists.txt b/cube_petit_text_to_speech/CMakeLists.txt new file mode 100644 index 0000000..79b3c32 --- /dev/null +++ b/cube_petit_text_to_speech/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 3.0.2) +project(cube_petit_text_to_speech) + +## Compile as C++11, supported in ROS Kinetic and newer +# add_compile_options(-std=c++11) + +find_package( + catkin REQUIRED + COMPONENTS roscpp + rospy + std_msgs + tf +) + +catkin_package( + CATKIN_DEPENDS + roscpp + rospy + std_msgs + tf +) + +########### +## Build ## +########### + +## Specify additional locations of header files +## Your package locations should be listed before other locations +include_directories( + # include + ${catkin_INCLUDE_DIRS} +) diff --git a/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch b/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch new file mode 100644 index 0000000..81fa2af --- /dev/null +++ b/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch @@ -0,0 +1,5 @@ + + + + + diff --git a/cube_petit_text_to_speech/package.xml b/cube_petit_text_to_speech/package.xml new file mode 100644 index 0000000..9e481b2 --- /dev/null +++ b/cube_petit_text_to_speech/package.xml @@ -0,0 +1,52 @@ + + + cube_petit_text_to_speech + 0.0.0 + The cube_petit_text_to_speech package + + + + + Airi Yokochi + + + + + + TODO + + + + + + + + + + + + + + + + + catkin + roscpp + rospy + std_msgs + tf + roscpp + rospy + std_msgs + tf + roscpp + rospy + std_msgs + + + + + + + + diff --git a/cube_petit_text_to_speech/scripts/jtalk.py b/cube_petit_text_to_speech/scripts/jtalk.py new file mode 100755 index 0000000..86301fe --- /dev/null +++ b/cube_petit_text_to_speech/scripts/jtalk.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import os +import re +import struct +import subprocess +import sys +import time +from datetime import datetime +from tokenize import String + +import rospy +from cube_speech.srv import CallSpeech +# +from sensor_msgs.msg import Joy +from std_msgs.msg import Bool +from std_msgs.msg import Float64 +from std_msgs.msg import Int32 +from std_msgs.msg import String + +# module_name, package_name, ClassName, method_name, +# ExceptionName, function_name, GLOBAL_CONSTANT_NAME +# global_var_name, instance_var_name, function_parameter_name, local_var_name. + +# init [TODO] Class +rospy.loginfo("------------------------------------------") +rospy.init_node('cube_petit_text_to_speech') + +rospy.wait_for_service('speech_service') +try: + speaker = rospy.ServiceProxy('speech_service', CallSpeech) +except rospy.ServiceException: + rospy.loginfo("Fail.") + +# 1Hz +rate = rospy.Rate(10) + +# instance_var_name [TODO] param or smtng easy +emotion = 'happiness' +pitch = 130 +speed = 100 +volume = 100 +speech_method = 'jtalk' + + +# Class method1: text_to_jtalk +def text_to_jtalk(phrase): + emotion = 'happiness' + rospy.loginfo('Speech Happily: ' + phrase) + result = speaker( + phrase, speech_method, emotion, None, pitch, speed, 100).result + if not result: + rospy.loginfo('Speech failed for some unknown reason') + else: + rospy.loginfo('Robot speech: {}'.format(phrase)) + # rospy.sleep(2.0) + + +def text_to_jtalk_sadness(phrase): + emotion = 'sadness' + rospy.loginfo('Speech Sadly: ' + phrase) + result = speaker( + phrase, speech_method, emotion, None, pitch, speed, 100).result + if not result: + rospy.loginfo('Speech failed for some unknown reason') + else: + rospy.loginfo('Robot speech: {}'.format(phrase)) + # rospy.sleep(2.0) + +def text_to_jtalk_shout(phrase): + emotion = 'shout' + rospy.loginfo('Speech Shout: ' + phrase) + result = speaker( + phrase, speech_method, emotion, None, pitch, speed, 100).result + if not result: + rospy.loginfo('Speech failed for some unknown reason') + else: + rospy.loginfo('Robot speech: {}'.format(phrase)) + # rospy.sleep(2.0) + + +julius_text = '' +battery_voltage = 0 +hight_height_flag = 0 +low_height_flag = 0 +middle_height_flag = 0 + +true_count = 0 +false_count = 0 +hotword_flag = 1 +motion_sensor_flag = 0 +imu_flag = 0 +mute_flag = 0 + + +get_julius_text_flag = 0 + +def julius_callback(phrase): + global julius_text + julius_text = phrase.data + rospy.loginfo("[jtalk.py(julius_callback)]:"+ julius_text) + +def imu_callback(data): + global hight_height_flag + global low_height_flag + global imu_flag + if(imu_flag > 0): + if(data.data > 50 and hight_height_flag == 0): + text_to_jtalk("高いよう") + hight_height_flag = 1 + elif(data.data < -50 and low_height_flag == 0): + text_to_jtalk("低いよう") + low_height_flag = 1 + elif(data.data <= 30 and data.data >= -30): + hight_height_flag = 0 + low_height_flag = 0 + + +def motion_sensor_callback(data): + global true_count + global false_count + global motion_sensor_flag + if(motion_sensor_flag > 0): + if(data.data == True): + true_count = true_count + 1 + if( true_count > 20 and false_count > 0): + true_count = 0 + false_count = 0 + text_to_jtalk("やっほー！来てくれてありがとう、仲良くしてね") + else: + false_count = false_count + 1 + +def battery_callback(data): + global battery_voltage + battery_voltage = data.data + + +def recog_and_talk(arg_text): + global hotword_flag + global julius_text + rospy.loginfo("[jtalk.py] recog_and_talk") + talk_text = '' + + if "コントローラ" in arg_text: + battery_status_result = subprocess.check_output('upower --show-info /org/freedesktop/UPower/devices/gaming_input_sony_controller_battery_88o03o4co2bo6fo06 | grep percentage| sed -e \'s/[^0-9]//g\'', shell=True) + num_str = str(battery_status_result) + num_str = re.sub(r'\D', '', num_str) + if num_str != '': + talk_text = "コントローラのバッテリ残量は" + num_str + "パーセントです" + text_to_jtalk(talk_text) + else: + talk_text = "コントローラが繋がっていません" + text_to_jtalk(talk_text) + elif ("何時" in arg_text) or ("時刻" in arg_text) or ("時間" in arg_text): + d = datetime.now() + talk_text = '今は%s月%s日、%s時%s分%s秒です。' % (d.month, d.day, d.hour, d.minute, d.second) + text_to_jtalk(talk_text) + elif "名前" in arg_text: + text_to_jtalk("僕の名前はキューブプチです。あなたの名前はなんですか") + julius_text = '' + rospy.sleep(5.5) + if julius_text != '': + talk_text = julius_text + "さんですね。素敵な名前ですね。" + text_to_jtalk(talk_text) + else: + text_to_jtalk("わかりませんでした。もう一度話しかけてね") + rospy.loginfo('[WARN] julius_text is none ') + elif ("電圧" in arg_text) or ("バッテリ" in arg_text): + talk_text = "バッテリの電圧は" + str(battery_voltage) + "です" + text_to_jtalk(talk_text) + elif ("扇風機" in arg_text) or ("ファン" in arg_text)or ("サーキュレーター" in arg_text): + talk_text = "おっけー" + text_to_jtalk(talk_text) + publish.publish(True) + else: + talk_text = julius_text + "ですね？" + text_to_jtalk(talk_text) + julius_text = '' + +def hotword_callback(data): + global hotword_flag + rospy.loginfo("hotword_callback") + global julius_text + + if hotword_flag == 1: + hotword_flag = 0 + if str(data.data) == "Cuboid": + text_to_jtalk('僕の名前はキューブプチです') + elif str(data.data) == "Cube-petit": + text_to_jtalk('はーい！なんですか？') + julius_text = '' + rospy.sleep(3.0) + count = 0 + listen_flag = 0 + while count < 5: + count += 1 + rospy.sleep(1.0) + if julius_text != '': + listen_flag = 1 + rospy.loginfo('julius_text:'+julius_text) + recog_and_talk(julius_text) + break + if listen_flag == 0: + text_to_jtalk("わかりませんでした。もう一度話しかけてね") + julius_text != '' + rospy.loginfo('[WARN] julius_text is none ') + hotword_flag = 1 + + +# Class method2: joyCallback [TODO] MUTEX +def callback(data): + global julius_text + global motion_sensor_flag + global imu_flag + global battery_voltage + global hotword_flag + global mute_flag + if data.buttons[0] == 1: # batu + rospy.loginfo("X") + text_to_jtalk('こんにちは') + # elif data.buttons[1] == 1: # maru + # rospy.loginfo("○") + elif data.buttons[2] == 1: # sankaku + rospy.loginfo("△") + text_to_jtalk('はいどうも') + # rate.sleep() + elif data.buttons[3] == 1: # sikaku + rospy.loginfo("□") + text_to_jtalk('ハローワールド！僕の名前はキューブプチです！') + # rate.sleep() + elif data.buttons[4] == 1: # L1 + rospy.loginfo("L1") + text_to_jtalk('こっち来てえ') + elif data.buttons[5] == 1: # R1 + rospy.loginfo("R1") + text_to_jtalk('ソラジロー先輩、どこですか〜？') + elif data.buttons[6] == 1: # L2 + rospy.loginfo("L2") + elif data.buttons[7] == 1: # R2 + rospy.loginfo("R2") + elif data.buttons[8] == 1: # SHARE + rospy.loginfo("SHARE") + text_to_jtalk('好きな野菜はなんですか〜？') + text_to_jtalk('ポンッ！') + + hotword_flag = 0 + julius_text = '' + yasai_text = '' + rospy.sleep(5.5) + julius_text = julius_text.rstrip('です') + julius_text = julius_text.rstrip('だよ') + if julius_text != '': + if julius_text == "キューブプチ": + text_to_jtalk("僕は食べられないよう！") + elif julius_text == "キューボイド": + text_to_jtalk("キューボイドは僕のお兄ちゃんの名前ですよ") + elif julius_text == "大根" or julius_text == "じゃがいも" or julius_text == "アスパラガス" or julius_text == "たけのこ": + # レンコン かぼちゃ アスパラ ブロッコリー + # アボカド + # たまねぎ + # じゃがいも + # そらまめ + # とうもろこし + # 白菜 + # キャベツ + # ピーマン + # ナス + # 筍 + # そら豆 + # えんどう豆 + # オクラ + yasai_text = julius_text + "は僕も大好きです！" + text_to_jtalk(yasai_text) + else: + yasai_text = julius_text + "ですね？" + text_to_jtalk(yasai_text) + julius_text = '' + else: + text_to_jtalk_sadness("わかりませんでした") + hotword_flag = 1 + + + elif data.buttons[9] == 1: # OPTION + hotword_flag = 0 + text_to_jtalk('音声認識モード。ポンッ！') + julius_text = '' + rospy.sleep(3.0) + count = 0 + listen_flag = 0 + while count < 5: + count += 1 + rospy.sleep(1.0) + if julius_text != '': + listen_flag = 1 + rospy.loginfo('julius_text:'+julius_text) + recog_and_talk(julius_text) + break + if listen_flag == 0: + text_to_jtalk("わかりませんでした。もう一度話しかけてね") + julius_text != '' + rospy.loginfo('[WARN] julius_text is none ') + hotword_flag = 1 + elif data.buttons[10] == 1: # PS + rospy.loginfo("PS Button") + if mute_flag == 0: + # マイクミュート [TODO]ミュート　ミュート解除 + text_to_jtalk("マイクをミュートします") + status = os.system('amixer -D pulse sget Capture') + rospy.loginfo('mute mic') + rospy.loginfo(str(status)) + rospy.loginfo(status) + mute_flag = 1 + elif mute_flag == 1: + text_to_jtalk("マイクのミュートを解除します") + status = os.system('amixer -D pulse sset Capture 80%') + mute_flag = 0 + elif data.buttons[11] == 1: # Left Stick + rospy.loginfo("Left Stick") + if(imu_flag==0): + text_to_jtalk("高さ検知機能オン") + imu_flag = 1 + else: + text_to_jtalk("高さ検知機能オフ") + imu_flag = 0 + elif data.buttons[12] == 1: # Right Stick + rospy.loginfo("Right Stick") + if(motion_sensor_flag==0): + text_to_jtalk("モーションセンサーオン") + motion_sensor_flag = 1 + else: + text_to_jtalk("モーションセンサーオフ") + motion_sensor_flag = 0 + elif data.axes[6] == 1.0: # Left Cross Button + rospy.loginfo("Left Cross Button") + text_to_jtalk('一緒に写真を撮ろう！') + elif data.axes[6] == -1.0: # Right Cross Button + rospy.loginfo("Right Cross Button") + text_to_jtalk_sadness('お腹が減ったよ、充電したいな') + elif data.axes[7] == 1.0: # Up Cross Button + rospy.loginfo("Up Cross Button") + text_to_jtalk('おかえりなさい！待ってたよ') + elif data.axes[7] == -1.0: # Down Cross Button + rospy.loginfo("Down Cross Button") + text_to_jtalk('お疲れ様！今日はどんな一日だった？') + + +# start + +# talk test +phrase = 'テストテスト。おはようございます。' +result = speaker( + phrase, speech_method, emotion, None, pitch, speed, volume).result +if not result: + rospy.loginfo('Speech failed for some unknown reason') +else: + rospy.loginfo('Robot speech: {}'.format(phrase)) + +# talk daytime +d = datetime.now() +phrase1 = '今は%s月%s日、%s時%s分%s秒です。起動しました' % (d.month, d.day, d.hour, d.minute, d.second) +result = speaker( + phrase1, speech_method, emotion, None, pitch, speed, volume).result +if not result: + rospy.loginfo('Speech failed for some unknown reason') +else: + rospy.loginfo('Robot speech: {}'.format(phrase1)) + +rospy.Subscriber("/joystick/joy",Joy,callback, queue_size=1) +rospy.Subscriber("/imu_height",Int32,imu_callback, queue_size=10) +rospy.Subscriber("/julius_result_text", String, julius_callback, queue_size=1) +rospy.Subscriber("/motion_sensor", Bool, motion_sensor_callback, queue_size=1) +rospy.Subscriber("/gazebo_battery_monitor/battery_voltage", Float64, battery_callback, queue_size=1) + +rospy.Subscriber("/hotword_detector/detect_word", String, hotword_callback, queue_size=1) + +publish = rospy.Publisher('/fan_on', Bool, queue_size=1) + +# status = os.system('amixer -D pulse sset Capture 0') +# rospy.loginfo('mute mic') + + +rospy.spin() diff --git a/cube_petit_text_to_speech/scripts/text_to_emotion.py b/cube_petit_text_to_speech/scripts/text_to_emotion.py new file mode 100755 index 0000000..4a4bff2 --- /dev/null +++ b/cube_petit_text_to_speech/scripts/text_to_emotion.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import json +import os +import re +import struct +import subprocess +import sys +import time +from datetime import datetime +from tokenize import String + +import rospy +from cube_speech.srv import CallSpeech +from mlask import MLAsk +# +from sensor_msgs.msg import Joy +from std_msgs.msg import Bool +from std_msgs.msg import Float64 +from std_msgs.msg import Int32 +from std_msgs.msg import String + +# module_name, package_name, ClassName, method_name, +# ExceptionName, function_name, GLOBAL_CONSTANT_NAME +# global_var_name, instance_var_name, function_parameter_name, local_var_name. + +# init +rospy.loginfo("------------------------------------------") +rospy.init_node('cube_petit_text_to_emoiton') + +emotion_analyzer = MLAsk() + +# wait for service +rospy.wait_for_service('speech_service') +try: + speaker = rospy.ServiceProxy('speech_service', CallSpeech) +except rospy.ServiceException: + rospy.loginfo("Fail.") + +# 1Hz +rate = rospy.Rate(10) + +# global var +julius_text = '' +emotion_flag = 1 + + +def text_to_emotion_callback(data): + if data.data: + emotion_flag = 1 + elif data.data: + emotion_flag = 0 + + +def julius_callback(phrase): + global julius_text + julius_text = phrase.data + if emotion_flag: + result_json = emotion_analyzer.analyze(julius_text) + parsed_data = json.dumps(result_json, indent=4) + rospy.loginfo(result_json) + if "emotion" in parsed_data: + text = str(result_json["emotion"]) + start_index = text.find('{') + if start_index != -1: + result = text[start_index:] # '{' から末尾までの部分を取得 + # rospy.loginfo(result) # '{' から始まる部分が出力されます + match = re.search(r"'([^']+)'", result) + if match: + extracted_string = match.group(1) + # rospy.loginfo(extracted_string) + publish_emotion.publish(extracted_string) + if "activation" in result_json: # 'NEUTRAL' 'mostly_ACTIVE' 'ACTIVE' 'mostly_PASSIVE' 'PASSIVE' + # rospy.loginfo(str(result_json["activation"])) + publish_activation.publish(str(result_json["activation"])) + if "orientation" in result_json: # 'NEUTRAL' 'POSITIVE' 'NEGATIVE' 'mostly_POSITIVE' 'mostly_NEGATIVE' + # rospy.loginfo(str(result_json["orientation"])) + publish_orientation.publish(str(result_json["orientation"])) + + +# start +rospy.Subscriber("/julius_result_text", String, julius_callback, queue_size=1) +rospy.Subscriber("/text_to_emotion_on", Bool, text_to_emotion_callback, queue_size=1) + +# 'yorokobi' 'ikari' 'aware' 'kowagari' 'haji' 'suki' 'iya' 'takaburi' 'yasuragi' 'odoroki' +publish_emotion = rospy.Publisher('/talk_to_emotion/emotion', String, queue_size=1) + +publish_activation = rospy.Publisher('/talk_to_emotion/activation', String, queue_size=1) +publish_orientation = rospy.Publisher('/talk_to_emotion/orientation', String, queue_size=1) + +rospy.spin() From 409ecaf19926c606cdbbba021fcdf266fae1fcbc Mon Sep 17 00:00:00 2001 From: AYokochi Date: Mon, 27 Nov 2023 17:41:21 +0900 Subject: [PATCH 2/3] comment text_to_emotion --- .../launch/cube_petit_text_to_speech.launch | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch b/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch index 81fa2af..b282ce3 100644 --- a/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch +++ b/cube_petit_text_to_speech/launch/cube_petit_text_to_speech.launch @@ -1,5 +1,10 @@ - - + + + + + From 5b2144c04b224d09ed4871d39819479487f67ab7 Mon Sep 17 00:00:00 2001 From: AYokochi Date: Tue, 4 Jun 2024 11:23:41 +0900 Subject: [PATCH 3/3] delete comment --- cube_petit_text_to_speech/package.xml | 34 +--------------------- cube_petit_text_to_speech/scripts/jtalk.py | 7 +---- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/cube_petit_text_to_speech/package.xml b/cube_petit_text_to_speech/package.xml index 9e481b2..1354ecd 100644 --- a/cube_petit_text_to_speech/package.xml +++ b/cube_petit_text_to_speech/package.xml @@ -3,33 +3,8 @@ cube_petit_text_to_speech 0.0.0 The cube_petit_text_to_speech package - - - - Airi Yokochi - - - - - - TODO - - - - - - - - - - - - - - - - + Apache2.0 catkin roscpp rospy @@ -42,11 +17,4 @@ roscpp rospy std_msgs - - - - - - - diff --git a/cube_petit_text_to_speech/scripts/jtalk.py b/cube_petit_text_to_speech/scripts/jtalk.py index 86301fe..1785bdf 100755 --- a/cube_petit_text_to_speech/scripts/jtalk.py +++ b/cube_petit_text_to_speech/scripts/jtalk.py @@ -36,14 +36,13 @@ # 1Hz rate = rospy.Rate(10) -# instance_var_name [TODO] param or smtng easy +# instance_var_name [TODO] param emotion = 'happiness' pitch = 130 speed = 100 volume = 100 speech_method = 'jtalk' - # Class method1: text_to_jtalk def text_to_jtalk(phrase): emotion = 'happiness' @@ -54,8 +53,6 @@ def text_to_jtalk(phrase): rospy.loginfo('Speech failed for some unknown reason') else: rospy.loginfo('Robot speech: {}'.format(phrase)) - # rospy.sleep(2.0) - def text_to_jtalk_sadness(phrase): emotion = 'sadness' @@ -66,7 +63,6 @@ def text_to_jtalk_sadness(phrase): rospy.loginfo('Speech failed for some unknown reason') else: rospy.loginfo('Robot speech: {}'.format(phrase)) - # rospy.sleep(2.0) def text_to_jtalk_shout(phrase): emotion = 'shout' @@ -77,7 +73,6 @@ def text_to_jtalk_shout(phrase): rospy.loginfo('Speech failed for some unknown reason') else: rospy.loginfo('Robot speech: {}'.format(phrase)) - # rospy.sleep(2.0) julius_text = ''