diff --git a/HomeAssistant.xcodeproj/project.pbxproj b/HomeAssistant.xcodeproj/project.pbxproj index bc502ec4c..c38f6394a 100644 --- a/HomeAssistant.xcodeproj/project.pbxproj +++ b/HomeAssistant.xcodeproj/project.pbxproj @@ -104,8 +104,6 @@ 11267D0925BBA9FE00F28E5C /* Updater.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 11267D0825BBA9FE00F28E5C /* Updater.test.swift */; }; 1127381C2622B6F300F5E312 /* DebugSettingsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127381B2622B6F300F5E312 /* DebugSettingsViewController.swift */; }; 1127383C2625512600F5E312 /* ButtonRowWithLoading.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127383B2625512600F5E312 /* ButtonRowWithLoading.swift */; }; - 1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; }; - 1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; }; 1130A5742751B29E00640E38 /* PerServerContainer.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5732751B29E00640E38 /* PerServerContainer.test.swift */; }; 1130A5762751BA1800640E38 /* Server.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5752751BA1800640E38 /* Server.test.swift */; }; 1130A5782751BDD900640E38 /* ServerManager.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5772751BDD900640E38 /* ServerManager.test.swift */; }; @@ -1472,7 +1470,6 @@ 1128FF38297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Frontend.strings; sourceTree = ""; }; 1128FF39297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/InfoPlist.strings; sourceTree = ""; }; 1128FF3A297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Localizable.strings; sourceTree = ""; }; - 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Locale+IntentLanguage.swift"; sourceTree = ""; }; 112B705A2526B1C500FEAA76 /* UpdateSensorsIntentHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UpdateSensorsIntentHandler.swift; sourceTree = ""; }; 1130A5732751B29E00640E38 /* PerServerContainer.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerServerContainer.test.swift; sourceTree = ""; }; 1130A5752751BA1800640E38 /* Server.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Server.test.swift; sourceTree = ""; }; @@ -5166,7 +5163,6 @@ 1165704F270188E4003906A7 /* URLComponents+WidgetAuthenticity.swift */, 116570762702B0F6003906A7 /* DiskCache.swift */, 1120C5832749C6350046C38B /* ServerProviding.swift */, - 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */, ); path = Common; sourceTree = ""; @@ -7170,7 +7166,6 @@ 114CBAE92839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */, 4251AABE2C6CE242004CCC9D /* MagicItemProvider.swift in Sources */, 42D3E4BE2C5D31E000444BE6 /* LocalNotificationDispatcher.swift in Sources */, - 1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */, 420AE9E12CA559FE0020E9CB /* Color+hex.swift in Sources */, 42D3E49D2C5BB88F00444BE6 /* WatchBatterySensor.swift in Sources */, 11C65CC1249838EB00D07FC7 /* StreamCameraResponse.swift in Sources */, @@ -7424,7 +7419,6 @@ D03D893B20E0B2E300D4F28D /* AppConstants.swift in Sources */, 119DE933263325C20099F7D8 /* IconDrawable+Settings.swift in Sources */, 114CBAE82839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */, - 1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */, D03D893520E0AEF100D4F28D /* Realm+Initialization.swift in Sources */, D0EEF2C9214D89A700D1D360 /* HAAPI+RequestHelpers.swift in Sources */, 428338442BA1BB4F004798C2 /* Spaces.swift in Sources */, diff --git a/Sources/App/Resources/Base.lproj/Intents.intentdefinition b/Sources/App/Resources/Base.lproj/Intents.intentdefinition index 258d9e8d8..8109310e4 100644 --- a/Sources/App/Resources/Base.lproj/Intents.intentdefinition +++ b/Sources/App/Resources/Base.lproj/Intents.intentdefinition @@ -9,11 +9,11 @@ INIntentDefinitionNamespace sI7YSe INIntentDefinitionSystemVersion - 24B5070a + 24C5089c INIntentDefinitionToolsBuildVersion - 16A242d + 16B40 INIntentDefinitionToolsVersion - 16.0 + 16.1 INIntents @@ -1861,10 +1861,17 @@ INIntentKeyParameter text INIntentLastParameterTag - 5 + 7 INIntentManagedParameterCombinations - text,server,language + server,text + + INIntentParameterCombinationSupportsBackgroundExecution + + INIntentParameterCombinationUpdatesLinked + + + text,server,pipeline INIntentParameterCombinationSupportsBackgroundExecution @@ -1880,7 +1887,7 @@ Assist INIntentParameterCombinations - text,server,language + text,server INIntentParameterCombinationIsPrimary @@ -1891,6 +1898,17 @@ INIntentParameterCombinationTitleID uqeIcc + text,server,pipeline + + INIntentParameterCombinationIsLinked + + INIntentParameterCombinationSupportsBackgroundExecution + + INIntentParameterCombinationTitle + Assist with "${text}" + INIntentParameterCombinationTitleID + 5b3s58 + INIntentParameters @@ -1962,65 +1980,102 @@ INIntentParameterConfigurable + INIntentParameterCustomDisambiguation + INIntentParameterDisplayName - Text + Pipeline INIntentParameterDisplayNameID - txfcnn + Nj6GCk INIntentParameterDisplayPriority 2 - INIntentParameterMetadata - - INIntentParameterMetadataCapitalization - Sentences - INIntentParameterMetadataDefaultValueID - adkcOI - INIntentParameterName - text + pipeline + INIntentParameterObjectType + IntentAssistPipeline + INIntentParameterObjectTypeNamespace + sI7YSe INIntentParameterPromptDialogs INIntentParameterPromptDialogCustom + INIntentParameterPromptDialogFormatString + Which pipeline? + INIntentParameterPromptDialogFormatStringID + qKFbLL INIntentParameterPromptDialogType Configuration INIntentParameterPromptDialogCustom + INIntentParameterPromptDialogFormatString + Which ${pipeline}? + INIntentParameterPromptDialogFormatStringID + 6wJumM INIntentParameterPromptDialogType Primary + + INIntentParameterPromptDialogCustom + + INIntentParameterPromptDialogFormatString + There are ${count} options matching ‘${pipeline}’. + INIntentParameterPromptDialogFormatStringID + wAhZcM + INIntentParameterPromptDialogType + DisambiguationIntroduction + + + INIntentParameterPromptDialogCustom + + INIntentParameterPromptDialogFormatString + Just to confirm, you wanted ‘${pipeline}’? + INIntentParameterPromptDialogFormatStringID + K6M7Jv + INIntentParameterPromptDialogType + Confirmation + + INIntentParameterRelationship + + INIntentParameterRelationshipParentName + server + INIntentParameterRelationshipPredicateName + HasAnyValue + + INIntentParameterSupportsDynamicEnumeration + + INIntentParameterSupportsResolution + INIntentParameterTag - 1 + 7 INIntentParameterType - String + Object INIntentParameterConfigurable INIntentParameterDisplayName - Language + Text INIntentParameterDisplayNameID - BafuI1 + txfcnn INIntentParameterDisplayPriority 3 + INIntentParameterMetadata + + INIntentParameterMetadataCapitalization + Sentences + INIntentParameterMetadataDefaultValueID + adkcOI + INIntentParameterName - language - INIntentParameterObjectType - IntentLanguage - INIntentParameterObjectTypeNamespace - sI7YSe + text INIntentParameterPromptDialogs INIntentParameterPromptDialogCustom - INIntentParameterPromptDialogFormatString - Which language? - INIntentParameterPromptDialogFormatStringID - QCGKRz INIntentParameterPromptDialogType Configuration @@ -2031,12 +2086,10 @@ Primary - INIntentParameterSupportsDynamicEnumeration - INIntentParameterTag - 5 + 1 INIntentParameterType - Object + String INIntentResponse diff --git a/Sources/Shared/Common/Locale+IntentLanguage.swift b/Sources/Shared/Common/Locale+IntentLanguage.swift deleted file mode 100644 index ec949688e..000000000 --- a/Sources/Shared/Common/Locale+IntentLanguage.swift +++ /dev/null @@ -1,15 +0,0 @@ -import Foundation - -extension Locale { - var asIntentLanguage: IntentLanguage { - .init(identifier: identifier, display: localizedString(forIdentifier: identifier) ?? identifier) - } - - var intentLanguages: [IntentLanguage] { - Locale.availableIdentifiers.map { - IntentLanguage(identifier: $0, display: localizedString(forIdentifier: $0) ?? $0) - }.sorted(by: { a, b in - a.displayString.localizedCaseInsensitiveCompare(b.displayString) == .orderedAscending - }) - } -} diff --git a/Sources/Shared/Intents/AssistIntentHandler.swift b/Sources/Shared/Intents/AssistIntentHandler.swift index f1ab0577e..1e13749af 100644 --- a/Sources/Shared/Intents/AssistIntentHandler.swift +++ b/Sources/Shared/Intents/AssistIntentHandler.swift @@ -5,6 +5,9 @@ import PromiseKit class AssistIntentHandler: NSObject, AssistIntentHandling { typealias Intent = AssistIntent + private var intentCompletion: ((AssistIntentResponse) -> Void)? + private var assistService: AssistService? + func resolveServer(for intent: Intent, with completion: @escaping (IntentServerResolutionResult) -> Void) { if let server = Current.servers.server(for: intent) { completion(.success(with: .init(server: server))) @@ -24,24 +27,6 @@ class AssistIntentHandler: NSObject, AssistIntentHandling { completion(.init(items: IntentServer.all), nil) } - func defaultLanguage(for intent: AssistIntent) -> IntentLanguage? { - Locale.current.asIntentLanguage - } - - func provideLanguageOptions( - for intent: AssistIntent, - with completion: @escaping ([IntentLanguage]?, Error?) -> Void - ) { - completion(Locale.current.intentLanguages, nil) - } - - func provideLanguageOptionsCollection( - for intent: AssistIntent, - with completion: @escaping (INObjectCollection?, Error?) -> Void - ) { - completion(.init(items: Locale.current.intentLanguages), nil) - } - func handle(intent: AssistIntent, completion: @escaping (AssistIntentResponse) -> Void) { guard let server = Current.servers.server(for: intent) else { completion(.failure(error: "no server provided")) @@ -56,34 +41,79 @@ class AssistIntentHandler: NSObject, AssistIntentHandling { return } - struct ConversationResponse: ImmutableMappable { - var speech: String + intentCompletion = completion + assistService = AssistService(server: server) + assistService?.delegate = self + assistService?.assist(source: .text(input: intent.text ?? "", pipelineId: intent.pipeline?.identifier ?? nil)) + } + + func resolvePipeline( + for intent: AssistIntent, + with completion: @escaping (IntentAssistPipelineResolutionResult) -> Void + ) { + guard let server = Current.servers.server(for: intent) else { + completion(.needsValue()) + return + } - init(map: Map) throws { - self.speech = try map.value("response.speech.plain.speech") + AssistService(server: server).fetchPipelines { response in + guard let pipelines = response?.pipelines else { + completion(.needsValue()) + return } + guard let result = pipelines.first(where: { pipeline in + pipeline.id == intent.pipeline?.identifier + }) else { + completion(.needsValue()) + return + } + completion(.success(with: .init(identifier: result.id, display: result.name))) } + } - Current.webhooks.sendEphemeral( - server: server, - request: .init( - type: "conversation_process", - data: [ - "text": intent.text, - "language": intent.language?.identifier ?? Locale.current.identifier, - ] - ) - ).map { (original: [String: Any]) -> (ConversationResponse, [String: Any]) in - let object: ConversationResponse = try Mapper().map(JSONObject: original) - return (object, original) - }.done { object, original in - Current.Log.info("finishing with \(object)") - let value = IntentAssistResult(identifier: nil, display: object.speech) - value.json = try String(decoding: JSONSerialization.data(withJSONObject: original), as: UTF8.self) - completion(.success(result: value)) - }.catch { error in - Current.Log.error("erroring with \(error)") - completion(.failure(error: error.localizedDescription)) + func providePipelineOptionsCollection( + for intent: AssistIntent, + with completion: @escaping (INObjectCollection?, (any Error)?) -> Void + ) { + guard let server = Current.servers.server(for: intent) else { + completion(.init(items: []), nil) + return + } + + AssistService(server: server).fetchPipelines { response in + guard let pipelines = response?.pipelines else { + completion(.init(items: []), nil) + return + } + completion(.init(items: pipelines.map({ pipeline in + IntentAssistPipeline(identifier: pipeline.id, display: pipeline.name) + })), nil) } } } + +extension AssistIntentHandler: AssistServiceDelegate { + func didReceiveEvent(_ event: AssistEvent) { + /* no-op */ + } + + func didReceiveSttContent(_ content: String) { + /* no-op */ + } + + func didReceiveIntentEndContent(_ content: String) { + intentCompletion?(.success(result: .init(identifier: nil, display: content))) + } + + func didReceiveGreenLightForAudioInput() { + /* no-op */ + } + + func didReceiveTtsMediaUrl(_ mediaUrl: URL) { + /* no-op */ + } + + func didReceiveError(code: String, message: String) { + intentCompletion?(.failure(error: "\(code) - \(message)")) + } +}