Skip to content

Commit

Permalink
Allow choosing pipeline when using Assist shortcut (#3242)
Browse files Browse the repository at this point in the history 
<!-- Thank you for submitting a Pull Request and helping to improve Home
Assistant. Please complete the following sections to help the processing
and review of your changes. Please do not delete anything from this
template. -->

## Summary
<!-- Provide a brief summary of the changes you have made and most
importantly what they aim to achieve -->

## Screenshots
<!-- If this is a user-facing change not in the frontend, please include
screenshots in light and dark mode. -->

![IMG_1064F5A7D40B-1](https://github.com/user-attachments/assets/51c26d26-48df-41d3-9c41-5c9661fd199f)

## Link to pull request in Documentation repository
<!-- Pull requests that add, change or remove functionality must have a
corresponding pull request in the Companion App Documentation repository
(https://github.com/home-assistant/companion.home-assistant). Please add
the number of this pull request after the "#" -->
Documentation: home-assistant/companion.home-assistant#

## Any other notes
<!-- If there is any other information of note, like if this Pull
Request is part of a bigger change, please include it here. -->
  • Loading branch information
@bgoncal
bgoncal authored Dec 6, 2024
1 parent bd45612 commit f7671eb
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 96 deletions.
6 changes: 0 additions & 6 deletions HomeAssistant.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@
11267D0925BBA9FE00F28E5C /* Updater.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 11267D0825BBA9FE00F28E5C /* Updater.test.swift */; };
1127381C2622B6F300F5E312 /* DebugSettingsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127381B2622B6F300F5E312 /* DebugSettingsViewController.swift */; };
1127383C2625512600F5E312 /* ButtonRowWithLoading.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127383B2625512600F5E312 /* ButtonRowWithLoading.swift */; };
1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; };
1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; };
1130A5742751B29E00640E38 /* PerServerContainer.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5732751B29E00640E38 /* PerServerContainer.test.swift */; };
1130A5762751BA1800640E38 /* Server.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5752751BA1800640E38 /* Server.test.swift */; };
1130A5782751BDD900640E38 /* ServerManager.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5772751BDD900640E38 /* ServerManager.test.swift */; };
Expand Down Expand Up @@ -1472,7 +1470,6 @@
1128FF38297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Frontend.strings; sourceTree = "<group>"; };
1128FF39297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/InfoPlist.strings; sourceTree = "<group>"; };
1128FF3A297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Localizable.strings; sourceTree = "<group>"; };
1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Locale+IntentLanguage.swift"; sourceTree = "<group>"; };
112B705A2526B1C500FEAA76 /* UpdateSensorsIntentHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UpdateSensorsIntentHandler.swift; sourceTree = "<group>"; };
1130A5732751B29E00640E38 /* PerServerContainer.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerServerContainer.test.swift; sourceTree = "<group>"; };
1130A5752751BA1800640E38 /* Server.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Server.test.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -5166,7 +5163,6 @@
1165704F270188E4003906A7 /* URLComponents+WidgetAuthenticity.swift */,
116570762702B0F6003906A7 /* DiskCache.swift */,
1120C5832749C6350046C38B /* ServerProviding.swift */,
1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */,
);
path = Common;
sourceTree = "<group>";
Expand Down Expand Up @@ -7170,7 +7166,6 @@
114CBAE92839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */,
4251AABE2C6CE242004CCC9D /* MagicItemProvider.swift in Sources */,
42D3E4BE2C5D31E000444BE6 /* LocalNotificationDispatcher.swift in Sources */,
1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */,
420AE9E12CA559FE0020E9CB /* Color+hex.swift in Sources */,
42D3E49D2C5BB88F00444BE6 /* WatchBatterySensor.swift in Sources */,
11C65CC1249838EB00D07FC7 /* StreamCameraResponse.swift in Sources */,
Expand Down Expand Up @@ -7424,7 +7419,6 @@
D03D893B20E0B2E300D4F28D /* AppConstants.swift in Sources */,
119DE933263325C20099F7D8 /* IconDrawable+Settings.swift in Sources */,
114CBAE82839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */,
1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */,
D03D893520E0AEF100D4F28D /* Realm+Initialization.swift in Sources */,
D0EEF2C9214D89A700D1D360 /* HAAPI+RequestHelpers.swift in Sources */,
428338442BA1BB4F004798C2 /* Spaces.swift in Sources */,
Expand Down
119 changes: 86 additions & 33 deletions Sources/App/Resources/Base.lproj/Intents.intentdefinition
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
<key>INIntentDefinitionNamespace</key>
<string>sI7YSe</string>
<key>INIntentDefinitionSystemVersion</key>
<string>24B5070a</string>
<string>24C5089c</string>
<key>INIntentDefinitionToolsBuildVersion</key>
<string>16A242d</string>
<string>16B40</string>
<key>INIntentDefinitionToolsVersion</key>
<string>16.0</string>
<string>16.1</string>
<key>INIntents</key>
<array>
<dict>
Expand Down Expand Up @@ -1861,10 +1861,17 @@
<key>INIntentKeyParameter</key>
<string>text</string>
<key>INIntentLastParameterTag</key>
<integer>5</integer>
<integer>7</integer>
<key>INIntentManagedParameterCombinations</key>
<dict>
<key>text,server,language</key>
<key>server,text</key>
<dict>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
<key>INIntentParameterCombinationUpdatesLinked</key>
<true/>
</dict>
<key>text,server,pipeline</key>
<dict>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
Expand All @@ -1880,7 +1887,7 @@
<string>Assist</string>
<key>INIntentParameterCombinations</key>
<dict>
<key>text,server,language</key>
<key>text,server</key>
<dict>
<key>INIntentParameterCombinationIsPrimary</key>
<true/>
Expand All @@ -1891,6 +1898,17 @@
<key>INIntentParameterCombinationTitleID</key>
<string>uqeIcc</string>
</dict>
<key>text,server,pipeline</key>
<dict>
<key>INIntentParameterCombinationIsLinked</key>
<true/>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
<key>INIntentParameterCombinationTitle</key>
<string>Assist with "${text}"</string>
<key>INIntentParameterCombinationTitleID</key>
<string>5b3s58</string>
</dict>
</dict>
<key>INIntentParameters</key>
<array>
Expand Down Expand Up @@ -1962,65 +1980,102 @@
<dict>
<key>INIntentParameterConfigurable</key>
<true/>
<key>INIntentParameterCustomDisambiguation</key>
<true/>
<key>INIntentParameterDisplayName</key>
<string>Text</string>
<string>Pipeline</string>
<key>INIntentParameterDisplayNameID</key>
<string>txfcnn</string>
<string>Nj6GCk</string>
<key>INIntentParameterDisplayPriority</key>
<integer>2</integer>
<key>INIntentParameterMetadata</key>
<dict>
<key>INIntentParameterMetadataCapitalization</key>
<string>Sentences</string>
<key>INIntentParameterMetadataDefaultValueID</key>
<string>adkcOI</string>
</dict>
<key>INIntentParameterName</key>
<string>text</string>
<string>pipeline</string>
<key>INIntentParameterObjectType</key>
<string>IntentAssistPipeline</string>
<key>INIntentParameterObjectTypeNamespace</key>
<string>sI7YSe</string>
<key>INIntentParameterPromptDialogs</key>
<array>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which pipeline?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>qKFbLL</string>
<key>INIntentParameterPromptDialogType</key>
<string>Configuration</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which ${pipeline}?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>6wJumM</string>
<key>INIntentParameterPromptDialogType</key>
<string>Primary</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>There are ${count} options matching ‘${pipeline}’.</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>wAhZcM</string>
<key>INIntentParameterPromptDialogType</key>
<string>DisambiguationIntroduction</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Just to confirm, you wanted ‘${pipeline}’?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>K6M7Jv</string>
<key>INIntentParameterPromptDialogType</key>
<string>Confirmation</string>
</dict>
</array>
<key>INIntentParameterRelationship</key>
<dict>
<key>INIntentParameterRelationshipParentName</key>
<string>server</string>
<key>INIntentParameterRelationshipPredicateName</key>
<string>HasAnyValue</string>
</dict>
<key>INIntentParameterSupportsDynamicEnumeration</key>
<true/>
<key>INIntentParameterSupportsResolution</key>
<true/>
<key>INIntentParameterTag</key>
<integer>1</integer>
<integer>7</integer>
<key>INIntentParameterType</key>
<string>String</string>
<string>Object</string>
</dict>
<dict>
<key>INIntentParameterConfigurable</key>
<true/>
<key>INIntentParameterDisplayName</key>
<string>Language</string>
<string>Text</string>
<key>INIntentParameterDisplayNameID</key>
<string>BafuI1</string>
<string>txfcnn</string>
<key>INIntentParameterDisplayPriority</key>
<integer>3</integer>
<key>INIntentParameterMetadata</key>
<dict>
<key>INIntentParameterMetadataCapitalization</key>
<string>Sentences</string>
<key>INIntentParameterMetadataDefaultValueID</key>
<string>adkcOI</string>
</dict>
<key>INIntentParameterName</key>
<string>language</string>
<key>INIntentParameterObjectType</key>
<string>IntentLanguage</string>
<key>INIntentParameterObjectTypeNamespace</key>
<string>sI7YSe</string>
<string>text</string>
<key>INIntentParameterPromptDialogs</key>
<array>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which language?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>QCGKRz</string>
<key>INIntentParameterPromptDialogType</key>
<string>Configuration</string>
</dict>
Expand All @@ -2031,12 +2086,10 @@
<string>Primary</string>
</dict>
</array>
<key>INIntentParameterSupportsDynamicEnumeration</key>
<true/>
<key>INIntentParameterTag</key>
<integer>5</integer>
<integer>1</integer>
<key>INIntentParameterType</key>
<string>Object</string>
<string>String</string>
</dict>
</array>
<key>INIntentResponse</key>
Expand Down
15 changes: 0 additions & 15 deletions Sources/Shared/Common/Locale+IntentLanguage.swift
View file

This file was deleted.

114 changes: 72 additions & 42 deletions Sources/Shared/Intents/AssistIntentHandler.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import PromiseKit
class AssistIntentHandler: NSObject, AssistIntentHandling {
typealias Intent = AssistIntent

private var intentCompletion: ((AssistIntentResponse) -> Void)?
private var assistService: AssistService?

func resolveServer(for intent: Intent, with completion: @escaping (IntentServerResolutionResult) -> Void) {
if let server = Current.servers.server(for: intent) {
completion(.success(with: .init(server: server)))
Expand All @@ -24,24 +27,6 @@ class AssistIntentHandler: NSObject, AssistIntentHandling {
completion(.init(items: IntentServer.all), nil)
}

func defaultLanguage(for intent: AssistIntent) -> IntentLanguage? {
Locale.current.asIntentLanguage
}

func provideLanguageOptions(
for intent: AssistIntent,
with completion: @escaping ([IntentLanguage]?, Error?) -> Void
) {
completion(Locale.current.intentLanguages, nil)
}

func provideLanguageOptionsCollection(
for intent: AssistIntent,
with completion: @escaping (INObjectCollection<IntentLanguage>?, Error?) -> Void
) {
completion(.init(items: Locale.current.intentLanguages), nil)
}

func handle(intent: AssistIntent, completion: @escaping (AssistIntentResponse) -> Void) {
guard let server = Current.servers.server(for: intent) else {
completion(.failure(error: "no server provided"))
Expand All @@ -56,34 +41,79 @@ class AssistIntentHandler: NSObject, AssistIntentHandling {
return
}

struct ConversationResponse: ImmutableMappable {
var speech: String
intentCompletion = completion
assistService = AssistService(server: server)
assistService?.delegate = self
assistService?.assist(source: .text(input: intent.text ?? "", pipelineId: intent.pipeline?.identifier ?? nil))
}

func resolvePipeline(
for intent: AssistIntent,
with completion: @escaping (IntentAssistPipelineResolutionResult) -> Void
) {
guard let server = Current.servers.server(for: intent) else {
completion(.needsValue())
return
}

init(map: Map) throws {
self.speech = try map.value("response.speech.plain.speech")
AssistService(server: server).fetchPipelines { response in
guard let pipelines = response?.pipelines else {
completion(.needsValue())
return
}
guard let result = pipelines.first(where: { pipeline in
pipeline.id == intent.pipeline?.identifier
}) else {
completion(.needsValue())
return
}
completion(.success(with: .init(identifier: result.id, display: result.name)))
}
}

Current.webhooks.sendEphemeral(
server: server,
request: .init(
type: "conversation_process",
data: [
"text": intent.text,
"language": intent.language?.identifier ?? Locale.current.identifier,
]
)
).map { (original: [String: Any]) -> (ConversationResponse, [String: Any]) in
let object: ConversationResponse = try Mapper().map(JSONObject: original)
return (object, original)
}.done { object, original in
Current.Log.info("finishing with \(object)")
let value = IntentAssistResult(identifier: nil, display: object.speech)
value.json = try String(decoding: JSONSerialization.data(withJSONObject: original), as: UTF8.self)
completion(.success(result: value))
}.catch { error in
Current.Log.error("erroring with \(error)")
completion(.failure(error: error.localizedDescription))
func providePipelineOptionsCollection(
for intent: AssistIntent,
with completion: @escaping (INObjectCollection<IntentAssistPipeline>?, (any Error)?) -> Void
) {
guard let server = Current.servers.server(for: intent) else {
completion(.init(items: []), nil)
return
}

AssistService(server: server).fetchPipelines { response in
guard let pipelines = response?.pipelines else {
completion(.init(items: []), nil)
return
}
completion(.init(items: pipelines.map({ pipeline in
IntentAssistPipeline(identifier: pipeline.id, display: pipeline.name)
})), nil)
}
}
}

extension AssistIntentHandler: AssistServiceDelegate {
func didReceiveEvent(_ event: AssistEvent) {
/* no-op */
}

func didReceiveSttContent(_ content: String) {
/* no-op */
}

func didReceiveIntentEndContent(_ content: String) {
intentCompletion?(.success(result: .init(identifier: nil, display: content)))
}

func didReceiveGreenLightForAudioInput() {
/* no-op */
}

func didReceiveTtsMediaUrl(_ mediaUrl: URL) {
/* no-op */
}

func didReceiveError(code: String, message: String) {
intentCompletion?(.failure(error: "\(code) - \(message)"))
}
}

0 comments on commit f7671eb

Please sign in to comment.