From 2c838a1c70338607998f47e0444aa4637f2fd078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Pantale=C3=A3o=20Gon=C3=A7alves?= Date: Wed, 10 Jul 2024 10:32:33 +0200 Subject: [PATCH] Improvements for Apple Watch Assist (#2839) ## Summary ## Screenshots ## Link to pull request in Documentation repository Documentation: home-assistant/companion.home-assistant# ## Any other notes --- Sources/App/Assist/AssistViewModel.swift | 6 + .../Resources/en.lproj/Localizable.strings | 1 + Sources/App/WatchCommunicatorService.swift | 83 +++++---- .../Assist/Settings/WatchAssistSettings.swift | 71 ++++---- .../Watch/Assist/WatchAssistService.swift | 157 +++++++++++------- .../Watch/Assist/WatchAssistView+Build.swift | 1 + .../Watch/Assist/WatchAssistView.swift | 72 +++----- .../Watch/Assist/WatchAssistViewModel.swift | 88 ++++++++-- .../Watch/Assist/WatchAudioRecorder.swift | 24 --- .../Extensions/Watch/Home/WatchHomeView.swift | 3 +- .../Watch/Home/WatchHomeViewModel.swift | 6 - .../Extensions/Watch/HostingController.swift | 2 +- .../Watch/WatchCommunicatorService.swift | 16 +- .../Intents/AssistInApp/AssistModel.swift | 4 + .../Intents/AssistInApp/AssistRequests.swift | 17 +- .../Intents/AssistInApp/AssistService.swift | 10 +- .../Shared/Resources/Swiftgen/Strings.swift | 4 + .../Watch/InteractiveImmediateMessages.swift | 1 + 18 files changed, 318 insertions(+), 248 deletions(-) diff --git a/Sources/App/Assist/AssistViewModel.swift b/Sources/App/Assist/AssistViewModel.swift index d3a036ac1..5c0125e18 100644 --- a/Sources/App/Assist/AssistViewModel.swift +++ b/Sources/App/Assist/AssistViewModel.swift @@ -205,6 +205,12 @@ extension AssistViewModel: AssistServiceDelegate { func didReceiveTtsMediaUrl(_ mediaUrl: URL) { audioPlayer.play(url: mediaUrl) } + + @MainActor + func didReceiveError(code: String, message: String) { + Current.Log.error("Assist error: \(code)") + appendToChat(.init(content: message, itemType: .error)) + } } extension AssistViewModel: AssistSessionDelegate { diff --git a/Sources/App/Resources/en.lproj/Localizable.strings b/Sources/App/Resources/en.lproj/Localizable.strings index 7bcd32023..f29201918 100644 --- a/Sources/App/Resources/en.lproj/Localizable.strings +++ b/Sources/App/Resources/en.lproj/Localizable.strings @@ -62,6 +62,7 @@ "assist.error.pipelines_response" = "Failed to obtain Assist pipelines, please check your pipelines configuration."; "assist.pipelines_picker.title" = "Assist Pipelines"; "assist.watch.mic_button.title" = "Tap to "; +"assist.watch.not_reachable.title" = "Assist requires iPhone connectivity. Your iPhone is currently unreachable."; "cancel_label" = "Cancel"; "carPlay.action.intro.item.body" = "Tap to continue on your iPhone"; "carPlay.action.intro.item.title" = "Create your first action"; diff --git a/Sources/App/WatchCommunicatorService.swift b/Sources/App/WatchCommunicatorService.swift index 8d111b51a..4ad432bb8 100644 --- a/Sources/App/WatchCommunicatorService.swift +++ b/Sources/App/WatchCommunicatorService.swift @@ -136,35 +136,27 @@ extension WatchCommunicatorService { return } - firstly { () -> Promise in - Promise { seal in - initAssistServiceIfNeeded(server: server).fetchPipelines { pipelinesResponse in - if let pipelines = pipelinesResponse?.pipelines, - let preferredPipeline = pipelinesResponse?.preferredPipeline { - message.reply(.init(identifier: responseIdentifier, content: [ - "pipelines": pipelines.map({ pipeline in - [ - "name": pipeline.name, - "id": pipeline.id, - ] - }), - "preferredPipeline": preferredPipeline, - ])) - seal.fulfill(()) - } else { - seal.reject(WatchAssistCommunicatorError.pipelinesFetchFailed) - } - } + initAssistServiceIfNeeded(server: server).fetchPipelines { pipelinesResponse in + if let pipelines = pipelinesResponse?.pipelines, + let preferredPipeline = pipelinesResponse?.preferredPipeline { + message.reply(.init(identifier: responseIdentifier, content: [ + "pipelines": pipelines.map({ pipeline in + [ + "name": pipeline.name, + "id": pipeline.id, + ] + }), + "preferredPipeline": preferredPipeline, + ])) + } else { + Current.Log + .error("Error during fetch Assist pipelines: \(WatchAssistCommunicatorError.pipelinesFetchFailed)") + message.reply(.init(identifier: responseIdentifier, content: ["error": true])) } - }.catch { err in - Current.Log.error("Error during fetch Assist pipelines: \(err)") - message.reply(.init(identifier: responseIdentifier, content: ["error": true])) } } private func assistAudioData(blob: Blob) { - let responseIdentifier = InteractiveImmediateResponses.assistAudioDataResponse.rawValue - let serverId = blob.metadata?["serverId"] as? String guard let server = Current.servers.all.first(where: { $0.identifier.rawValue == serverId }) ?? Current .servers.all.first else { @@ -173,28 +165,18 @@ extension WatchCommunicatorService { return } - firstly { [weak self] () -> Promise in - Promise { seal in - let pipelineId = blob.metadata?["pipelineId"] as? String ?? "" - guard let self, let sampleRate = blob.metadata?["sampleRate"] as? Double else { - let errorMessage = "No sample rate received in message \(blob.identifier)" - Current.Log.error(errorMessage) - return - } - let audioData = blob.content - self.pendingAudioData = audioData - self.initAssistServiceIfNeeded(server: server).assist(source: .audio( - pipelineId: pipelineId, - audioSampleRate: sampleRate - )) - DispatchQueue.global().asyncAfter(deadline: .now() + 3) { - seal.fulfill(()) - } - } - }.catch { err in - let errorMessage = "Error during fetch Assist pipelines: \(err)" - Current.Log.warning(errorMessage) + let pipelineId = blob.metadata?["pipelineId"] as? String + guard let sampleRate = blob.metadata?["sampleRate"] as? Double else { + let errorMessage = "No sample rate received in message \(blob.identifier)" + Current.Log.error(errorMessage) + return } + let audioData = blob.content + pendingAudioData = audioData + initAssistServiceIfNeeded(server: server).assist(source: .audio( + pipelineId: pipelineId, + audioSampleRate: sampleRate + )) } private func initAssistServiceIfNeeded(server: Server) -> AssistServiceProtocol { @@ -262,6 +244,17 @@ extension WatchCommunicatorService: AssistServiceDelegate { ) sendMessage(message: message) } + + func didReceiveError(code: String, message: String) { + let message = ImmediateMessage( + identifier: InteractiveImmediateResponses.assistError.rawValue, + content: [ + "code": code, + "message": message, + ] + ) + sendMessage(message: message) + } } // MARK: - ServerObserver diff --git a/Sources/Extensions/Watch/Assist/Settings/WatchAssistSettings.swift b/Sources/Extensions/Watch/Assist/Settings/WatchAssistSettings.swift index a3c902b75..cc74ced49 100644 --- a/Sources/Extensions/Watch/Assist/Settings/WatchAssistSettings.swift +++ b/Sources/Extensions/Watch/Assist/Settings/WatchAssistSettings.swift @@ -2,47 +2,54 @@ import Shared import SwiftUI struct WatchAssistSettings: View { - @EnvironmentObject var assistService: WatchAssistService + @StateObject private var assistService: WatchAssistService + + init(assistService: WatchAssistService) { + self._assistService = .init(wrappedValue: assistService) + } var body: some View { ScrollView { VStack(spacing: Spaces.two) { - VStack { - Text(L10n.Settings.ConnectionSection.servers) - Picker(selection: $assistService.selectedServer) { - ForEach($assistService.servers.wrappedValue, id: \.identifier.rawValue) { server in - Text(server.info.name) - .tag(server.identifier.rawValue) + if !assistService.servers.isEmpty, assistService.selectedServer != nil { + VStack { + Text(L10n.Settings.ConnectionSection.servers) + Picker(selection: $assistService.selectedServer) { + ForEach(assistService.servers, id: \.identifier.rawValue) { server in + Text(server.info.name) + .tag(server.identifier.rawValue) + } + } label: { + EmptyView() } - } label: { - EmptyView() - } - .modify { - if #available(watchOS 9, *) { - $0.pickerStyle(.navigationLink) - } else { - $0.pickerStyle(.wheel) - .frame(height: 100) + .modify { + if #available(watchOS 9, *) { + $0.pickerStyle(.navigationLink) + } else { + $0.pickerStyle(.wheel) + .frame(height: 100) + } } } } - - VStack { - Text(L10n.Assist.PipelinesPicker.title) - Picker(selection: $assistService.preferredPipeline) { - ForEach($assistService.pipelines.wrappedValue, id: \.id) { pipeline in - Text(pipeline.name) - .tag(pipeline.id) + if !assistService.isFetchingPipeline, !assistService.pipelines.isEmpty { + VStack { + Text(L10n.Assist.PipelinesPicker.title) + Picker(selection: $assistService.preferredPipeline) { + ForEach(assistService.pipelines, id: \.id) { pipeline in + Text(pipeline.name) + .tag(pipeline.id) + } + } label: { + EmptyView() } - } label: { - EmptyView() - } - .modify { - if #available(watchOS 9, *) { - $0.pickerStyle(.navigationLink) - } else { - $0.pickerStyle(.wheel) - .frame(height: 100) + .modify { + if #available(watchOS 9, *) { + $0.pickerStyle(.navigationLink) + } else { + $0.pickerStyle(.wheel) + .frame(height: 100) + } } } } diff --git a/Sources/Extensions/Watch/Assist/WatchAssistService.swift b/Sources/Extensions/Watch/Assist/WatchAssistService.swift index 0c79dc93c..2e34e1a4c 100644 --- a/Sources/Extensions/Watch/Assist/WatchAssistService.swift +++ b/Sources/Extensions/Watch/Assist/WatchAssistService.swift @@ -6,31 +6,77 @@ import Shared final class WatchAssistService: ObservableObject { @Published var servers: [Server] = [] - @Published var selectedServer: String = "" + @Published var selectedServer: String = "" { + didSet { + pipelines = [] + preferredPipeline = "" + + // Fetch new pipelines in case server changes manually + if !oldValue.isEmpty { + fetchPipelines(completion: { _ in }) + } + } + } @Published var pipelines: [WatchPipeline] = [] @Published var preferredPipeline: String = "" + @Published var deviceReachable = false + @Published var isFetchingPipeline = false private let watchPreferredServerUserDefaultsKey = "watch-preferred-server-id" - private var cancellable: AnyCancellable? + private var reachabilityObservation: Observation? init() { - Current.servers.add(observer: self) - self.cancellable = $selectedServer.sink { [weak self] newSelectedServer in - guard let self else { return } - UserDefaults().setValue(newSelectedServer, forKey: watchPreferredServerUserDefaultsKey) - self.preferredPipeline = "" - loadPipelines(serverId: newSelectedServer) { _ in } - } setupServers() + setupReachability() } deinit { - cancellable?.cancel() + endRoutine() + } + + func endRoutine() { + if let reachabilityObservation { + Reachability.unobserve(reachabilityObservation) + self.reachabilityObservation = nil + } } func fetchPipelines(completion: @escaping (Bool) -> Void) { - loadPipelines(completion: completion) + guard deviceReachable, !selectedServer.isEmpty else { + completion(false) + return + } + isFetchingPipeline = true + Current.Log.verbose("Signaling fetch Assist pipelines via phone") + let actionMessage = InteractiveImmediateMessage( + identifier: InteractiveImmediateMessages.assistPipelinesFetch.rawValue, + content: ["serverId": selectedServer], + reply: { [weak self] message in + Current.Log.verbose("Received reply dictionary \(message)") + if let pipelines = message.content["pipelines"] as? [[String: String]] { + self?.updatePipelines( + pipelines, + preferredPipeline: message.content["preferredPipeline"] as? String + ) + completion(true) + } else { + completion(false) + } + self?.runInMainThread { + self?.isFetchingPipeline = false + } + } + ) + + Current.Log + .verbose( + "Sending \(InteractiveImmediateMessages.assistPipelinesFetch.rawValue) message \(actionMessage)" + ) + Communicator.shared.send(actionMessage, errorHandler: { error in + Current.Log.error("Received error when sending immediate message \(error)") + completion(false) + }) } func assist(audioURL: URL, sampleRate: Double, completion: @escaping (Error?) -> Void) { @@ -44,14 +90,23 @@ final class WatchAssistService: ObservableObject { try FileManager.default.removeItem(at: audioURL) Current.Log.verbose("Signaling Assist audio data") + + var metadata: [String: Any] = [ + "sampleRate": sampleRate, + ] + + if !preferredPipeline.isEmpty { + metadata["pipelineId"] = preferredPipeline + } + + if !selectedServer.isEmpty { + metadata["serverId"] = selectedServer + } + let blob = Blob( identifier: InteractiveImmediateMessages.assistAudioData.rawValue, content: audioData, - metadata: [ - "serverId": selectedServer, - "pipelineId": preferredPipeline, - "sampleRate": sampleRate, - ] + metadata: metadata ) Current.Log.verbose("Sending \(blob.identifier)") @@ -64,52 +119,14 @@ final class WatchAssistService: ObservableObject { completion(error) } } - - // Extra message just to wake up iPhone from the background to process Blob above - Communicator.shared.send(ImmediateMessage(identifier: "wakeup")) } catch { Current.Log.error("Watch assist failed: \(error.localizedDescription)") completion(error) } } - private func loadPipelines(serverId: String? = nil, completion: @escaping (Bool) -> Void) { - let serverId = serverId ?? selectedServer - guard Communicator.shared.currentReachability == .immediatelyReachable else { - completion(false) - return - } - - Current.Log.verbose("Signaling fetch Assist pipelines via phone") - let actionMessage = InteractiveImmediateMessage( - identifier: InteractiveImmediateMessages.assistPipelinesFetch.rawValue, - content: ["serverId": serverId], - reply: { message in - Current.Log.verbose("Received reply dictionary \(message)") - if let pipelines = message.content["pipelines"] as? [[String: String]] { - self.updatePipelines( - pipelines, - preferredPipeline: message.content["preferredPipeline"] as? String ?? "" - ) - completion(true) - } else { - completion(false) - } - } - ) - - Current.Log - .verbose( - "Sending \(InteractiveImmediateMessages.assistPipelinesFetch.rawValue) message \(actionMessage)" - ) - Communicator.shared.send(actionMessage, errorHandler: { error in - Current.Log.error("Received error when sending immediate message \(error)") - completion(false) - }) - } - - private func updatePipelines(_ pipelines: [[String: String]], preferredPipeline: String) { - DispatchQueue.main.async { [weak self] in + private func updatePipelines(_ pipelines: [[String: String]], preferredPipeline: String?) { + runInMainThread { [weak self] in guard let self else { return } self.pipelines = pipelines.compactMap({ pipelineRawValue in guard let id = pipelineRawValue["id"], let name = pipelineRawValue["name"] else { @@ -120,9 +137,7 @@ final class WatchAssistService: ObservableObject { name: name ) }) - if self.preferredPipeline.isEmpty { - self.preferredPipeline = preferredPipeline - } + self.preferredPipeline = preferredPipeline ?? "" } } @@ -132,13 +147,27 @@ final class WatchAssistService: ObservableObject { servers.first(where: { $0.identifier.rawValue == preferredServer }) != nil { selectedServer = preferredServer } else { - selectedServer = Current.servers.all.first?.identifier.rawValue ?? "" + if let server = Current.servers.all.first?.identifier.rawValue { + selectedServer = server + } else { + selectedServer = "" + Current.Log.error("Watch Assist: No server available, this can't happen") + } } } -} -extension WatchAssistService: ServerObserver { - func serversDidChange(_ serverManager: any Shared.ServerManager) { - setupServers() + private func setupReachability() { + reachabilityObservation = Reachability.observe { [weak self] _ in + DispatchQueue.main.async { + self?.deviceReachable = Communicator.shared.currentReachability == .immediatelyReachable + } + } + deviceReachable = Communicator.shared.currentReachability == .immediatelyReachable + } + + private func runInMainThread(completion: @escaping () -> Void) { + DispatchQueue.main.async { + completion() + } } } diff --git a/Sources/Extensions/Watch/Assist/WatchAssistView+Build.swift b/Sources/Extensions/Watch/Assist/WatchAssistView+Build.swift index 9f3260ede..05cd7e6e7 100644 --- a/Sources/Extensions/Watch/Assist/WatchAssistView+Build.swift +++ b/Sources/Extensions/Watch/Assist/WatchAssistView+Build.swift @@ -5,6 +5,7 @@ extension WatchAssistView { static func build() -> WatchAssistView { let viewModel = WatchAssistViewModel( audioRecorder: WatchAudioRecorder(), + audioPlayer: AudioPlayer(), immediateCommunicatorService: .shared ) return WatchAssistView(viewModel: viewModel) diff --git a/Sources/Extensions/Watch/Assist/WatchAssistView.swift b/Sources/Extensions/Watch/Assist/WatchAssistView.swift index c0bce1317..908116ae9 100644 --- a/Sources/Extensions/Watch/Assist/WatchAssistView.swift +++ b/Sources/Extensions/Watch/Assist/WatchAssistView.swift @@ -3,7 +3,6 @@ import SwiftUI struct WatchAssistView: View { @StateObject private var viewModel: WatchAssistViewModel - @EnvironmentObject private var assistService: WatchAssistService /// Used when there are multiple server @State private var showSettings = false @@ -30,7 +29,7 @@ struct WatchAssistView: View { .handGestureShortcut(.primaryAction) */ .onTapGesture { - viewModel.assist(assistService) + viewModel.assist() } .modify { if #available(watchOS 10, *) { @@ -46,10 +45,10 @@ struct WatchAssistView: View { } } .onAppear { - initialRoutine() + viewModel.initialRoutine() } .onDisappear { - viewModel.stopRecording() + viewModel.endRoutine() } .onChange(of: viewModel.state) { newValue in // TODO: On watchOS 10 this can be replaced by '.sensoryFeedback' modifier @@ -72,21 +71,10 @@ struct WatchAssistView: View { } } .fullScreenCover(isPresented: $showSettings) { - WatchAssistSettings() + WatchAssistSettings(assistService: viewModel.assistService) } .onReceive(NotificationCenter.default.publisher(for: AssistDefaultComplication.launchNotification)) { _ in - initialRoutine() - } - } - - private func initialRoutine() { - viewModel.state = .loading - if assistService.pipelines.isEmpty { - assistService.fetchPipelines { _ in - viewModel.assist(assistService) - } - } else { - viewModel.assist(assistService) + viewModel.initialRoutine() } } @@ -111,47 +99,25 @@ struct WatchAssistView: View { @ViewBuilder private var pipelineSelector: some View { - if assistService.pipelines.count > 1 || assistService.servers.count > 1, - let firstPipelineName = assistService.pipelines - .first(where: { $0.id == assistService.preferredPipeline })?.name, - let firstPipelineNameChar = firstPipelineName.first { - Button { - if assistService.servers.count > 1 { - showSettings = true - } else { - showPipelinesPicker = true - } - } label: { - HStack { - if #available(watchOS 10, *) { - Text(String(firstPipelineNameChar)) - } else { - // When watchS below 10, this item has more space available - Text(firstPipelineName) - } - Image(systemName: "chevron.down") - .font(.system(size: 8)) - } - .padding(.horizontal) - } - .confirmationDialog(L10n.Assist.PipelinesPicker.title, isPresented: $showPipelinesPicker) { - ForEach(assistService.pipelines, id: \.id) { pipeline in - Button { - assistService.preferredPipeline = pipeline.id - } label: { - Text(pipeline.name) - } - } - } + Button { + showSettings = true + } label: { + Image(systemName: "gear") } } @ViewBuilder private var micButton: some View { if ![.loading, .recording].contains(viewModel.state), !viewModel.showChatLoader { - HStack { - Text(L10n.Assist.Watch.MicButton.title) - Image(systemName: "mic.fill") + HStack(spacing: .zero) { + if viewModel.assistService.deviceReachable { + Text(L10n.Assist.Watch.MicButton.title) + Image(systemName: "mic.fill") + } else { + Image(systemName: "iphone.slash") + .foregroundStyle(.red) + .padding(.trailing) + } } .font(.system(size: 11)) .foregroundStyle(.gray) @@ -189,7 +155,7 @@ struct WatchAssistView: View { @ViewBuilder private var micRecording: some View { Button(action: { - viewModel.assist(assistService) + viewModel.assist() }, label: { if #available(watchOS 10.0, *) { Image(systemName: "waveform.circle.fill") diff --git a/Sources/Extensions/Watch/Assist/WatchAssistViewModel.swift b/Sources/Extensions/Watch/Assist/WatchAssistViewModel.swift index d721d2833..21439fbab 100644 --- a/Sources/Extensions/Watch/Assist/WatchAssistViewModel.swift +++ b/Sources/Extensions/Watch/Assist/WatchAssistViewModel.swift @@ -21,34 +21,75 @@ final class WatchAssistViewModel: ObservableObject { @Published var showChatLoader = false private let audioRecorder: any WatchAudioRecorderProtocol + private let audioPlayer: any AudioPlayerProtocol private let immediateCommunicatorService: ImmediateCommunicatorService - /// Provided via environment object and received by 'assist' method - private var assistService: WatchAssistService? + @Published var assistService: WatchAssistService init( audioRecorder: any WatchAudioRecorderProtocol, + audioPlayer: any AudioPlayerProtocol, immediateCommunicatorService: ImmediateCommunicatorService ) { self.audioRecorder = audioRecorder self.immediateCommunicatorService = immediateCommunicatorService + self.assistService = WatchAssistService() + self.audioPlayer = audioPlayer audioRecorder.delegate = self immediateCommunicatorService.addObserver(.init(delegate: self)) } deinit { + endRoutine() + } + + func initialRoutine() { + state = .loading + guard !assistService.selectedServer.isEmpty else { + fatalError("Server can't be nil") + } + if assistService.pipelines.isEmpty || assistService.preferredPipeline.isEmpty { + Current.Log.info("Watch Assist: pipelines list is empty, trying to fetch pipelines") + assistService.fetchPipelines { [weak self] success in + Current.Log + .info("Watch Assist: Pipelines fetch done, result: \(success), moving on with assist command") + if success { + self?.assist() + } else { + self?.state = .idle + } + } + } else { + Current.Log.info("Watch Assist: pipelines list exist, moving on with assist command") + assist() + } + } + + func endRoutine() { + stopRecording() + assistService.endRoutine() immediateCommunicatorService.removeObserver(self) } - func assist(_ assistService: WatchAssistService) { - self.assistService = assistService - audioRecorder.startRecording() + func assist() { + if assistService.deviceReachable { + // Extra message just to wake up iPhone from the background + Communicator.shared.send(ImmediateMessage(identifier: "wakeup")) + audioRecorder.startRecording() + } else { + state = .idle + showUnreacheableMessage() + } } func stopRecording() { audioRecorder.stopRecording() } + private func showUnreacheableMessage() { + chatItems.append(.init(content: L10n.Assist.Watch.NotReachable.title, itemType: .error)) + } + private func showChatLoader(show: Bool) { DispatchQueue.main.async { [weak self] in self?.showChatLoader = show @@ -62,8 +103,12 @@ final class WatchAssistViewModel: ObservableObject { } private func sendAudioData(audioURL: URL, audioSampleRate: Double) { + guard assistService.deviceReachable else { + showUnreacheableMessage() + return + } showChatLoader(show: true) - assistService?.assist(audioURL: audioURL, sampleRate: audioSampleRate) { [weak self] error in + assistService.assist(audioURL: audioURL, sampleRate: audioSampleRate) { [weak self] error in if let error { Current.Log.error("Failed to assist from watch error: \(error.localizedDescription)") self?.updateState(state: .idle) @@ -82,28 +127,42 @@ final class WatchAssistViewModel: ObservableObject { self?.showChatLoader = false } } + + private func runInMainThread(completion: @escaping () -> Void) { + DispatchQueue.main.async { + completion() + } + } } extension WatchAssistViewModel: WatchAudioRecorderDelegate { @MainActor func didStartRecording() { - state = .recording + runInMainThread { [weak self] in + self?.state = .recording + } } @MainActor func didStopRecording() { - state = .waitingForPipelineResponse + runInMainThread { [weak self] in + self?.state = .waitingForPipelineResponse + } } @MainActor func didFinishRecording(audioURL: URL, audioSampleRate: Double) { sendAudioData(audioURL: audioURL, audioSampleRate: audioSampleRate) - state = .waitingForPipelineResponse + runInMainThread { [weak self] in + self?.state = .waitingForPipelineResponse + } } func didFailRecording(error: any Error) { Current.Log.error("Failed to record Assist audio in watch App: \(error.localizedDescription)") - state = .idle + runInMainThread { [weak self] in + self?.state = .idle + } } } @@ -111,4 +170,13 @@ extension WatchAssistViewModel: ImmediateCommunicatorServiceDelegate { func didReceiveChatItem(_ item: AssistChatItem) { appendChatItem(item) } + + func didReceiveTTS(url: URL) { + audioPlayer.play(url: url) + } + + func didReceiveError(code: String, message: String) { + Current.Log.error("Watch Assist error: \(code)") + appendChatItem(.init(content: message, itemType: .error)) + } } diff --git a/Sources/Extensions/Watch/Assist/WatchAudioRecorder.swift b/Sources/Extensions/Watch/Assist/WatchAudioRecorder.swift index 4d60d6050..ef049f834 100644 --- a/Sources/Extensions/Watch/Assist/WatchAudioRecorder.swift +++ b/Sources/Extensions/Watch/Assist/WatchAudioRecorder.swift @@ -17,9 +17,6 @@ protocol WatchAudioRecorderProtocol: ObservableObject { final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol { private var audioRecorder: AVAudioRecorder? - private var silenceTimer: Timer? - private let silenceThreshold: TimeInterval = 3.0 - private let silenceLevel: Float = -50.0 private var audioSampleRate: Double? weak var delegate: WatchAudioRecorderDelegate? @@ -57,7 +54,6 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol { Current.Log.verbose("Using audio sample rate \(String(describing: audioSampleRate))") audioRecorder?.record() - startMonitoringAudioLevels() delegate?.didStartRecording() } catch { delegate?.didFailRecording(error: error) @@ -67,8 +63,6 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol { func stopRecording() { audioRecorder?.stop() audioRecorder = nil - silenceTimer?.invalidate() - silenceTimer = nil delegate?.didStopRecording() } @@ -76,24 +70,6 @@ final class WatchAudioRecorder: NSObject, WatchAudioRecorderProtocol { let sharedGroupContainerDirectory = Constants.AppGroupContainer return sharedGroupContainerDirectory.appendingPathComponent("assist.wav") } - - private func startMonitoringAudioLevels() { - silenceTimer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in - guard let self, let audioRecorder else { return } - audioRecorder.updateMeters() - - let averagePower = audioRecorder.averagePower(forChannel: 1) - if averagePower < silenceLevel { - silenceTimer?.invalidate() - silenceTimer = Timer - .scheduledTimer(withTimeInterval: silenceThreshold, repeats: false) { [weak self] _ in - self?.stopRecording() - } - } else { - silenceTimer?.invalidate() - } - } - } } extension WatchAudioRecorder: AVAudioRecorderDelegate { diff --git a/Sources/Extensions/Watch/Home/WatchHomeView.swift b/Sources/Extensions/Watch/Home/WatchHomeView.swift index 0c200d3a5..301993d1c 100644 --- a/Sources/Extensions/Watch/Home/WatchHomeView.swift +++ b/Sources/Extensions/Watch/Home/WatchHomeView.swift @@ -25,7 +25,6 @@ struct WatchHomeView: View where ViewModel: WatchHomeViewModelProtoco } .fullScreenCover(isPresented: $showAssist, content: { WatchAssistView.build() - .environmentObject(viewModel.assistService) }) .onReceive(NotificationCenter.default.publisher(for: AssistDefaultComplication.launchNotification)) { _ in showAssist = true @@ -56,7 +55,7 @@ struct WatchHomeView: View where ViewModel: WatchHomeViewModelProtoco Button(action: { showAssist = true }, label: { - Image(uiImage: MaterialDesignIcons.microphoneIcon.image( + Image(uiImage: MaterialDesignIcons.messageProcessingOutlineIcon.image( ofSize: .init(width: 24, height: 24), color: Asset.Colors.haPrimary.color )) diff --git a/Sources/Extensions/Watch/Home/WatchHomeViewModel.swift b/Sources/Extensions/Watch/Home/WatchHomeViewModel.swift index 7155c73f2..8108f3093 100644 --- a/Sources/Extensions/Watch/Home/WatchHomeViewModel.swift +++ b/Sources/Extensions/Watch/Home/WatchHomeViewModel.swift @@ -15,7 +15,6 @@ struct WatchActionItem: Equatable { protocol WatchHomeViewModelProtocol: ObservableObject { var actions: [WatchActionItem] { get set } - var assistService: WatchAssistService { get set } func onAppear() func onDisappear() func runActionId(_ actionId: String, completion: @escaping (Bool) -> Void) @@ -36,15 +35,10 @@ enum WatchSendError: Error { final class WatchHomeViewModel: WatchHomeViewModelProtocol { @Published var actions: [WatchActionItem] = [] - @Published var assistService: WatchAssistService private var actionsToken: NotificationToken? private var realmActions: [Action] = [] - init(assistService: WatchAssistService) { - self.assistService = assistService - } - func onAppear() { setupActionsObservation() } diff --git a/Sources/Extensions/Watch/HostingController.swift b/Sources/Extensions/Watch/HostingController.swift index 105047127..5b3cd5707 100644 --- a/Sources/Extensions/Watch/HostingController.swift +++ b/Sources/Extensions/Watch/HostingController.swift @@ -3,6 +3,6 @@ import SwiftUI final class HostingController: WKHostingController> { override var body: WatchHomeView { - WatchHomeView(viewModel: .init(assistService: .init())) + WatchHomeView(viewModel: .init()) } } diff --git a/Sources/Extensions/Watch/WatchCommunicatorService.swift b/Sources/Extensions/Watch/WatchCommunicatorService.swift index 2bff6c458..f311e3ce9 100644 --- a/Sources/Extensions/Watch/WatchCommunicatorService.swift +++ b/Sources/Extensions/Watch/WatchCommunicatorService.swift @@ -8,6 +8,8 @@ struct ImmediateCommunicatorServiceObserver { protocol ImmediateCommunicatorServiceDelegate: AnyObject { func didReceiveChatItem(_ item: AssistChatItem) + func didReceiveTTS(url: URL) + func didReceiveError(code: String, message: String) } final class ImmediateCommunicatorService { @@ -42,7 +44,19 @@ final class ImmediateCommunicatorService { } observers.forEach({ $0.delegate?.didReceiveChatItem(AssistChatItem(content: content, itemType: .output)) }) case .assistTTSResponse: - break + guard let audioURLString = message.content["mediaURL"] as? String, + let audioURL = URL(string: audioURLString) else { + Current.Log.error("Received assistTTSResponse without valid media URL") + return + } + observers.forEach({ $0.delegate?.didReceiveTTS(url: audioURL) }) + case .assistError: + guard let code = message.content["code"] as? String, + let message = message.content["message"] as? String else { + Current.Log.error("Received assistError without valid code/message") + return + } + observers.forEach({ $0.delegate?.didReceiveError(code: code, message: message) }) default: break } diff --git a/Sources/Shared/Intents/AssistInApp/AssistModel.swift b/Sources/Shared/Intents/AssistInApp/AssistModel.swift index 26acf0f96..203abac28 100644 --- a/Sources/Shared/Intents/AssistInApp/AssistModel.swift +++ b/Sources/Shared/Intents/AssistInApp/AssistModel.swift @@ -90,6 +90,8 @@ public struct AssistResponse: HADataDecodable { public let runnerData: RunnerData? public let sttOutput: SttOutput? public let ttsOutput: TtsOutput? + public let code: String? + public let message: String? public init(data: HAData) throws { self.pipeline = try? data.decode("data") @@ -98,6 +100,8 @@ public struct AssistResponse: HADataDecodable { self.runnerData = try? data.decode("runner_data") self.sttOutput = try? data.decode("stt_output") self.ttsOutput = try? data.decode("tts_output") + self.code = try? data.decode("code") + self.message = try? data.decode("message") } public struct SttOutput: HADataDecodable { diff --git a/Sources/Shared/Intents/AssistInApp/AssistRequests.swift b/Sources/Shared/Intents/AssistInApp/AssistRequests.swift index 11818c553..b4077ee2b 100644 --- a/Sources/Shared/Intents/AssistInApp/AssistRequests.swift +++ b/Sources/Shared/Intents/AssistInApp/AssistRequests.swift @@ -2,50 +2,55 @@ import Foundation import HAKit public enum AssistRequests { + static var runCommand = "assist_pipeline/run" public static func assistByVoiceTypedSubscription( - preferredPipelineId: String, + preferredPipelineId: String?, audioSampleRate: Double, conversationId: String?, hassDeviceId: String? ) -> HATypedSubscription { var data: [String: Any] = [ - "pipeline": preferredPipelineId, "start_stage": "stt", "end_stage": "tts", "input": [ "sample_rate": audioSampleRate, ], ] + if let preferredPipelineId { + data["pipeline"] = preferredPipelineId + } if let conversationId { data["conversation_id"] = conversationId } if let hassDeviceId { data["device_id"] = hassDeviceId } - return .init(request: .init(type: .webSocket("assist_pipeline/run"), data: data)) + return .init(request: .init(type: .webSocket(runCommand), data: data)) } public static func assistByTextTypedSubscription( - preferredPipelineId: String, + preferredPipelineId: String?, inputText: String, conversationId: String?, hassDeviceId: String? ) -> HATypedSubscription { var data: [String: Any] = [ - "pipeline": preferredPipelineId, "start_stage": "intent", "end_stage": "intent", "input": [ "text": inputText, ], ] + if let preferredPipelineId { + data["pipeline"] = preferredPipelineId + } if let conversationId { data["conversation_id"] = conversationId } if let hassDeviceId { data["device_id"] = hassDeviceId } - return .init(request: .init(type: .webSocket("assist_pipeline/run"), data: data)) + return .init(request: .init(type: .webSocket(runCommand), data: data)) } public static var fetchPipelinesTypedRequest: HATypedRequest { diff --git a/Sources/Shared/Intents/AssistInApp/AssistService.swift b/Sources/Shared/Intents/AssistInApp/AssistService.swift index 1947ddb46..e79501b62 100644 --- a/Sources/Shared/Intents/AssistInApp/AssistService.swift +++ b/Sources/Shared/Intents/AssistInApp/AssistService.swift @@ -16,11 +16,12 @@ public protocol AssistServiceDelegate: AnyObject { func didReceiveIntentEndContent(_ content: String) func didReceiveGreenLightForAudioInput() func didReceiveTtsMediaUrl(_ mediaUrl: URL) + func didReceiveError(code: String, message: String) } public enum AssistSource: Equatable { - case text(input: String, pipelineId: String) - case audio(pipelineId: String, audioSampleRate: Double) + case text(input: String, pipelineId: String?) + case audio(pipelineId: String?, audioSampleRate: Double) public static func == (lhs: AssistSource, rhs: AssistSource) -> Bool { switch (lhs, rhs) { @@ -104,7 +105,7 @@ public final class AssistService: AssistServiceProtocol { _ = connection.send(.init(type: .sttData(.init(rawValue: sttBinaryHandlerId)))) } - private func assistWithAudio(pipelineId: String, audioSampleRate: Double) { + private func assistWithAudio(pipelineId: String?, audioSampleRate: Double) { lastPipelineIdUsed = pipelineId connection.subscribe(to: AssistRequests.assistByVoiceTypedSubscription( preferredPipelineId: pipelineId, @@ -118,7 +119,7 @@ public final class AssistService: AssistServiceProtocol { } } - private func assistWithText(input: String, pipelineId: String) { + private func assistWithText(input: String, pipelineId: String?) { lastPipelineIdUsed = pipelineId connection.subscribe(to: AssistRequests.assistByTextTypedSubscription( preferredPipelineId: pipelineId, @@ -174,6 +175,7 @@ public final class AssistService: AssistServiceProtocol { case .error: sttBinaryHandlerId = nil Current.Log.error("Received error while interating with Assist: \(data)") + delegate?.didReceiveError(code: data.data?.code ?? "-1", message: data.data?.message ?? "Unknown error") cancellable.cancel() } } diff --git a/Sources/Shared/Resources/Swiftgen/Strings.swift b/Sources/Shared/Resources/Swiftgen/Strings.swift index ac1a544db..d603089fa 100644 --- a/Sources/Shared/Resources/Swiftgen/Strings.swift +++ b/Sources/Shared/Resources/Swiftgen/Strings.swift @@ -296,6 +296,10 @@ public enum L10n { /// Tap to public static var title: String { return L10n.tr("Localizable", "assist.watch.mic_button.title") } } + public enum NotReachable { + /// Assist requires iPhone connectivity. Your iPhone is currently unreachable. + public static var title: String { return L10n.tr("Localizable", "assist.watch.not_reachable.title") } + } } } diff --git a/Sources/Shared/Watch/InteractiveImmediateMessages.swift b/Sources/Shared/Watch/InteractiveImmediateMessages.swift index fc700315d..4ba256e04 100644 --- a/Sources/Shared/Watch/InteractiveImmediateMessages.swift +++ b/Sources/Shared/Watch/InteractiveImmediateMessages.swift @@ -15,4 +15,5 @@ public enum InteractiveImmediateResponses: String, CaseIterable { case assistSTTResponse case assistIntentEndResponse case assistTTSResponse + case assistError }