Skip to content

Commit

Permalink
Implement more fine-grained typing notices sending
Browse files Browse the repository at this point in the history
The previous approach (implemented in dd1dd78) was simple
(send typing notices for as long as the "controller" is running),
but this proved to be overly simplistic and unable to handle edge-cases:

- in multi-user rooms (or rooms with a prefix requirement), the bot
  used to send a typing notice while "working", but its work consisted
  of ignoring the message. So it then sent a "not typing" notice.
  This is wasteful and otherwise problematic - certain clients (like nheko)
  do not handle this "race" well.

- certain reactions (anything other than 🗣️ right now) are meant to be
  ignored. There's no point in doing the same "typing / not typing"
  dance

- there are other instances where the bot may do work, but doesn't (due
  to configuration or lack of capabilities)

This new more fine-grained implementation of typing notices aims to:

- only send a typing notice if actual "slow work" will be done

- avoid stopping & restarting typing notices (wasteful) if a chain of work is to
  be performed (processing voice messages and doing speech-to-text +
  text-generation + ...). Rather, maintaining typing notice sending
  throughout
  • Loading branch information
spantaleev committed Sep 14, 2024
1 parent 509f683 commit 3b25b92
Show file tree
Hide file tree
Showing 8 changed files with 30 additions and 17 deletions.
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ base64 = "0.22.*"
# We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
matrix-sdk = { version = "0.7.1", default-features = false }
mxidwc = "1.0.*"
mxlink = "1.2.*"
mxlink = ">=1.2.1"
etke_openai_api_rust = "0.1.*"
quick_cache = "0.6.*"
regex = "1.10.*"
Expand Down
9 changes: 9 additions & 0 deletions src/bot/implementation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use mxlink::matrix_sdk::Room;

use mxlink::{
InitConfig, LoginConfig, LoginCredentials, LoginEncryption, MatrixLink, PersistenceConfig,
TypingNoticeGuard,
};

use mxlink::helpers::account_data_config::{
Expand Down Expand Up @@ -210,6 +211,14 @@ impl Bot {
.await
}

pub(crate) async fn start_typing_notice(&self, room: &Room) -> TypingNoticeGuard {
self.inner
.matrix_link
.rooms()
.start_typing_notice(room)
.await
}

pub async fn start(&self) -> anyhow::Result<()> {
self.rooms().attach_event_handlers().await;
self.messaging().attach_event_handlers().await;
Expand Down
7 changes: 0 additions & 7 deletions src/bot/messaging.rs
Original file line number Diff line number Diff line change
Expand Up @@ -317,13 +317,6 @@ impl Messaging {

let start_time = std::time::Instant::now();

let _typing_notice_guard = self
.bot
.matrix_link()
.rooms()
.start_typing_notice(message_context.room())
.await;

let event_span = tracing::error_span!("message_controller", ?controller_type);

crate::controller::dispatch_controller(&controller_type, &message_context, &self.bot)
Expand Down
7 changes: 0 additions & 7 deletions src/bot/reacting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -244,13 +244,6 @@ impl Reacting {

tracing::info!("Handling reaction via reaction controller");

let _typing_notice_guard = self
.bot
.matrix_link()
.rooms()
.start_typing_notice(message_context.room())
.await;

let event_span = tracing::error_span!("reaction_controller");

crate::controller::reaction::handle(
Expand Down
14 changes: 14 additions & 0 deletions src/controller/chat_completion/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ pub async fn handle(
) -> anyhow::Result<()> {
let mut original_message_is_audio = false;

let mut _typing_notice_guard: Option<mxlink::TypingNoticeGuard> = None;

let speech_to_text_flow_type = message_context
.room_config_context()
.speech_to_text_flow_type();
Expand Down Expand Up @@ -71,6 +73,10 @@ pub async fn handle(
}
};

if _typing_notice_guard.is_none() {
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
}

let Some(speech_to_text_created_event_id_result) =
handle_stage_speech_to_text(bot, message_context, audio_content, response_type).await
else {
Expand All @@ -96,6 +102,10 @@ pub async fn handle(
.room_config_context()
.should_auto_text_generate(original_message_is_audio)
{
if _typing_notice_guard.is_none() {
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
}

let speech_to_text_created_event_id_reaction_event_id =
if let Some(speech_to_text_created_event_id) = speech_to_text_created_event_id {
let reaction_event_response = bot
Expand Down Expand Up @@ -213,6 +223,10 @@ pub async fn handle(

match text_to_speech_stage_params {
Some(TextToSpeechParams::Perform(text_to_speech_eligible_payload, response_type)) => {
if _typing_notice_guard.is_none() {
_typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await);
}

let _tts_result = generate_and_send_tts_for_message(
bot,
matrix_link.clone(),
Expand Down
2 changes: 2 additions & 0 deletions src/controller/image/generation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ pub async fn handle_sticker(
return Ok(());
};

let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await;

let span = tracing::debug_span!(
"sticker_generation",
agent_id = agent.identifier().as_string()
Expand Down
2 changes: 2 additions & 0 deletions src/controller/reaction/text_to_speech.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ pub(super) async fn handle(
return Ok(());
};

let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await;

crate::controller::utils::text_to_speech::generate_and_send_tts_for_message(
bot,
matrix_link,
Expand Down

0 comments on commit 3b25b92

Please sign in to comment.