Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow choosing pipeline when using Assist shortcut #3242

Merged
merged 2 commits into from
Dec 6, 2024
Merged

Allow choosing pipeline when using Assist shortcut #3242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
592e183
Allow choosing pipeline in Assist shortcut
bgoncal Dec 6, 2024
312e186
Remove language from Assist shortcut
bgoncal Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions HomeAssistant.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@
11267D0925BBA9FE00F28E5C /* Updater.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 11267D0825BBA9FE00F28E5C /* Updater.test.swift */; };
1127381C2622B6F300F5E312 /* DebugSettingsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127381B2622B6F300F5E312 /* DebugSettingsViewController.swift */; };
1127383C2625512600F5E312 /* ButtonRowWithLoading.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1127383B2625512600F5E312 /* ButtonRowWithLoading.swift */; };
1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; };
1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */; };
1130A5742751B29E00640E38 /* PerServerContainer.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5732751B29E00640E38 /* PerServerContainer.test.swift */; };
1130A5762751BA1800640E38 /* Server.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5752751BA1800640E38 /* Server.test.swift */; };
1130A5782751BDD900640E38 /* ServerManager.test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1130A5772751BDD900640E38 /* ServerManager.test.swift */; };
Expand Down Expand Up @@ -1472,7 +1470,6 @@
1128FF38297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Frontend.strings; sourceTree = "<group>"; };
1128FF39297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/InfoPlist.strings; sourceTree = "<group>"; };
1128FF3A297E5F7D00BAAFD9 /* ml */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ml; path = ml.lproj/Localizable.strings; sourceTree = "<group>"; };
1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Locale+IntentLanguage.swift"; sourceTree = "<group>"; };
112B705A2526B1C500FEAA76 /* UpdateSensorsIntentHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UpdateSensorsIntentHandler.swift; sourceTree = "<group>"; };
1130A5732751B29E00640E38 /* PerServerContainer.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerServerContainer.test.swift; sourceTree = "<group>"; };
1130A5752751BA1800640E38 /* Server.test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Server.test.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -5166,7 +5163,6 @@
1165704F270188E4003906A7 /* URLComponents+WidgetAuthenticity.swift */,
116570762702B0F6003906A7 /* DiskCache.swift */,
1120C5832749C6350046C38B /* ServerProviding.swift */,
1128FF3B297F49D900BAAFD9 /* Locale+IntentLanguage.swift */,
);
path = Common;
sourceTree = "<group>";
Expand Down Expand Up @@ -7170,7 +7166,6 @@
114CBAE92839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */,
4251AABE2C6CE242004CCC9D /* MagicItemProvider.swift in Sources */,
42D3E4BE2C5D31E000444BE6 /* LocalNotificationDispatcher.swift in Sources */,
1128FF3D297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */,
420AE9E12CA559FE0020E9CB /* Color+hex.swift in Sources */,
42D3E49D2C5BB88F00444BE6 /* WatchBatterySensor.swift in Sources */,
11C65CC1249838EB00D07FC7 /* StreamCameraResponse.swift in Sources */,
Expand Down Expand Up @@ -7424,7 +7419,6 @@
D03D893B20E0B2E300D4F28D /* AppConstants.swift in Sources */,
119DE933263325C20099F7D8 /* IconDrawable+Settings.swift in Sources */,
114CBAE82839E49E00A9BAFF /* CustomServerTrustManager.swift in Sources */,
1128FF3C297F49D900BAAFD9 /* Locale+IntentLanguage.swift in Sources */,
D03D893520E0AEF100D4F28D /* Realm+Initialization.swift in Sources */,
D0EEF2C9214D89A700D1D360 /* HAAPI+RequestHelpers.swift in Sources */,
428338442BA1BB4F004798C2 /* Spaces.swift in Sources */,
Expand Down
119 changes: 86 additions & 33 deletions Sources/App/Resources/Base.lproj/Intents.intentdefinition
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
<key>INIntentDefinitionNamespace</key>
<string>sI7YSe</string>
<key>INIntentDefinitionSystemVersion</key>
<string>24B5070a</string>
<string>24C5089c</string>
<key>INIntentDefinitionToolsBuildVersion</key>
<string>16A242d</string>
<string>16B40</string>
<key>INIntentDefinitionToolsVersion</key>
<string>16.0</string>
<string>16.1</string>
<key>INIntents</key>
<array>
<dict>
Expand Down Expand Up @@ -1861,10 +1861,17 @@
<key>INIntentKeyParameter</key>
<string>text</string>
<key>INIntentLastParameterTag</key>
<integer>5</integer>
<integer>7</integer>
<key>INIntentManagedParameterCombinations</key>
<dict>
<key>text,server,language</key>
<key>server,text</key>
<dict>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
<key>INIntentParameterCombinationUpdatesLinked</key>
<true/>
</dict>
<key>text,server,pipeline</key>
<dict>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
Expand All @@ -1880,7 +1887,7 @@
<string>Assist</string>
<key>INIntentParameterCombinations</key>
<dict>
<key>text,server,language</key>
<key>text,server</key>
<dict>
<key>INIntentParameterCombinationIsPrimary</key>
<true/>
Expand All @@ -1891,6 +1898,17 @@
<key>INIntentParameterCombinationTitleID</key>
<string>uqeIcc</string>
</dict>
<key>text,server,pipeline</key>
<dict>
<key>INIntentParameterCombinationIsLinked</key>
<true/>
<key>INIntentParameterCombinationSupportsBackgroundExecution</key>
<true/>
<key>INIntentParameterCombinationTitle</key>
<string>Assist with "${text}"</string>
<key>INIntentParameterCombinationTitleID</key>
<string>5b3s58</string>
</dict>
</dict>
<key>INIntentParameters</key>
<array>
Expand Down Expand Up @@ -1962,65 +1980,102 @@
<dict>
<key>INIntentParameterConfigurable</key>
<true/>
<key>INIntentParameterCustomDisambiguation</key>
<true/>
<key>INIntentParameterDisplayName</key>
<string>Text</string>
<string>Pipeline</string>
<key>INIntentParameterDisplayNameID</key>
<string>txfcnn</string>
<string>Nj6GCk</string>
<key>INIntentParameterDisplayPriority</key>
<integer>2</integer>
<key>INIntentParameterMetadata</key>
<dict>
<key>INIntentParameterMetadataCapitalization</key>
<string>Sentences</string>
<key>INIntentParameterMetadataDefaultValueID</key>
<string>adkcOI</string>
</dict>
<key>INIntentParameterName</key>
<string>text</string>
<string>pipeline</string>
<key>INIntentParameterObjectType</key>
<string>IntentAssistPipeline</string>
<key>INIntentParameterObjectTypeNamespace</key>
<string>sI7YSe</string>
<key>INIntentParameterPromptDialogs</key>
<array>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which pipeline?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>qKFbLL</string>
<key>INIntentParameterPromptDialogType</key>
<string>Configuration</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which ${pipeline}?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>6wJumM</string>
<key>INIntentParameterPromptDialogType</key>
<string>Primary</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>There are ${count} options matching ‘${pipeline}’.</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>wAhZcM</string>
<key>INIntentParameterPromptDialogType</key>
<string>DisambiguationIntroduction</string>
</dict>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Just to confirm, you wanted ‘${pipeline}’?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>K6M7Jv</string>
<key>INIntentParameterPromptDialogType</key>
<string>Confirmation</string>
</dict>
</array>
<key>INIntentParameterRelationship</key>
<dict>
<key>INIntentParameterRelationshipParentName</key>
<string>server</string>
<key>INIntentParameterRelationshipPredicateName</key>
<string>HasAnyValue</string>
</dict>
<key>INIntentParameterSupportsDynamicEnumeration</key>
<true/>
<key>INIntentParameterSupportsResolution</key>
<true/>
<key>INIntentParameterTag</key>
<integer>1</integer>
<integer>7</integer>
<key>INIntentParameterType</key>
<string>String</string>
<string>Object</string>
</dict>
<dict>
<key>INIntentParameterConfigurable</key>
<true/>
<key>INIntentParameterDisplayName</key>
<string>Language</string>
<string>Text</string>
<key>INIntentParameterDisplayNameID</key>
<string>BafuI1</string>
<string>txfcnn</string>
<key>INIntentParameterDisplayPriority</key>
<integer>3</integer>
<key>INIntentParameterMetadata</key>
<dict>
<key>INIntentParameterMetadataCapitalization</key>
<string>Sentences</string>
<key>INIntentParameterMetadataDefaultValueID</key>
<string>adkcOI</string>
</dict>
<key>INIntentParameterName</key>
<string>language</string>
<key>INIntentParameterObjectType</key>
<string>IntentLanguage</string>
<key>INIntentParameterObjectTypeNamespace</key>
<string>sI7YSe</string>
<string>text</string>
<key>INIntentParameterPromptDialogs</key>
<array>
<dict>
<key>INIntentParameterPromptDialogCustom</key>
<true/>
<key>INIntentParameterPromptDialogFormatString</key>
<string>Which language?</string>
<key>INIntentParameterPromptDialogFormatStringID</key>
<string>QCGKRz</string>
<key>INIntentParameterPromptDialogType</key>
<string>Configuration</string>
</dict>
Expand All @@ -2031,12 +2086,10 @@
<string>Primary</string>
</dict>
</array>
<key>INIntentParameterSupportsDynamicEnumeration</key>
<true/>
<key>INIntentParameterTag</key>
<integer>5</integer>
<integer>1</integer>
<key>INIntentParameterType</key>
<string>Object</string>
<string>String</string>
</dict>
</array>
<key>INIntentResponse</key>
Expand Down
15 changes: 0 additions & 15 deletions Sources/Shared/Common/Locale+IntentLanguage.swift
View file
Open in desktop

This file was deleted.

114 changes: 72 additions & 42 deletions Sources/Shared/Intents/AssistIntentHandler.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import PromiseKit
class AssistIntentHandler: NSObject, AssistIntentHandling {
typealias Intent = AssistIntent

private var intentCompletion: ((AssistIntentResponse) -> Void)?
private var assistService: AssistService?

func resolveServer(for intent: Intent, with completion: @escaping (IntentServerResolutionResult) -> Void) {
if let server = Current.servers.server(for: intent) {
completion(.success(with: .init(server: server)))
Expand All @@ -24,24 +27,6 @@ class AssistIntentHandler: NSObject, AssistIntentHandling {
completion(.init(items: IntentServer.all), nil)
}

func defaultLanguage(for intent: AssistIntent) -> IntentLanguage? {
Locale.current.asIntentLanguage
}

func provideLanguageOptions(
for intent: AssistIntent,
with completion: @escaping ([IntentLanguage]?, Error?) -> Void
) {
completion(Locale.current.intentLanguages, nil)
}

func provideLanguageOptionsCollection(
for intent: AssistIntent,
with completion: @escaping (INObjectCollection<IntentLanguage>?, Error?) -> Void
) {
completion(.init(items: Locale.current.intentLanguages), nil)
}

func handle(intent: AssistIntent, completion: @escaping (AssistIntentResponse) -> Void) {
guard let server = Current.servers.server(for: intent) else {
completion(.failure(error: "no server provided"))
Expand All @@ -56,34 +41,79 @@ class AssistIntentHandler: NSObject, AssistIntentHandling {
return
}

struct ConversationResponse: ImmutableMappable {
var speech: String
intentCompletion = completion
assistService = AssistService(server: server)
assistService?.delegate = self
assistService?.assist(source: .text(input: intent.text ?? "", pipelineId: intent.pipeline?.identifier ?? nil))
}

func resolvePipeline(
for intent: AssistIntent,
with completion: @escaping (IntentAssistPipelineResolutionResult) -> Void
) {
guard let server = Current.servers.server(for: intent) else {
completion(.needsValue())
return
}

init(map: Map) throws {
self.speech = try map.value("response.speech.plain.speech")
AssistService(server: server).fetchPipelines { response in
guard let pipelines = response?.pipelines else {
completion(.needsValue())
return
}
guard let result = pipelines.first(where: { pipeline in
pipeline.id == intent.pipeline?.identifier
}) else {
completion(.needsValue())
return
}
completion(.success(with: .init(identifier: result.id, display: result.name)))
}
}

Current.webhooks.sendEphemeral(
server: server,
request: .init(
type: "conversation_process",
data: [
"text": intent.text,
"language": intent.language?.identifier ?? Locale.current.identifier,
]
)
).map { (original: [String: Any]) -> (ConversationResponse, [String: Any]) in
let object: ConversationResponse = try Mapper().map(JSONObject: original)
return (object, original)
}.done { object, original in
Current.Log.info("finishing with \(object)")
let value = IntentAssistResult(identifier: nil, display: object.speech)
value.json = try String(decoding: JSONSerialization.data(withJSONObject: original), as: UTF8.self)
completion(.success(result: value))
}.catch { error in
Current.Log.error("erroring with \(error)")
completion(.failure(error: error.localizedDescription))
func providePipelineOptionsCollection(
for intent: AssistIntent,
with completion: @escaping (INObjectCollection<IntentAssistPipeline>?, (any Error)?) -> Void
) {
guard let server = Current.servers.server(for: intent) else {
completion(.init(items: []), nil)
return
}

AssistService(server: server).fetchPipelines { response in
guard let pipelines = response?.pipelines else {
completion(.init(items: []), nil)
return
}
completion(.init(items: pipelines.map({ pipeline in
IntentAssistPipeline(identifier: pipeline.id, display: pipeline.name)
})), nil)
}
}
}

extension AssistIntentHandler: AssistServiceDelegate {
func didReceiveEvent(_ event: AssistEvent) {
/* no-op */
}

func didReceiveSttContent(_ content: String) {
/* no-op */
}

func didReceiveIntentEndContent(_ content: String) {
intentCompletion?(.success(result: .init(identifier: nil, display: content)))
}

func didReceiveGreenLightForAudioInput() {
/* no-op */
}

func didReceiveTtsMediaUrl(_ mediaUrl: URL) {
/* no-op */
}

func didReceiveError(code: String, message: String) {
intentCompletion?(.failure(error: "\(code) - \(message)"))
}
}
Loading