Skip to content

Commit

Permalink
Add Open AI integration (#1717)
Browse files Browse the repository at this point in the history
  • Loading branch information
jozefKruszynski authored Jan 1, 2024
1 parent af1b06c commit ba0a34d
Show file tree
Hide file tree
Showing 9 changed files with 308 additions and 10 deletions.
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,18 @@ MA requires a 64bit Operating System and a minimum of 2GB of RAM on the physical
NOTE: You need to set-up the players and music sources within Music Assistant itself.
If you are running Music Assistant in docker, you need to access the webinterface at http://youripaddress:8095, when running the Home Assistant add-on, you can access the webinterface from the add-on (and even show that in the sidebar).

## OpenAI features

During [Chapter 5 of "Year of the Voice"](https://www.youtube.com/live/djEkgoS5dDQ?si=pt8-qYH3PTpsnOq9&t=3699), [JLo](https://blog.jlpouffier.fr/chatgpt-powered-music-search-engine-on-a-local-voice-assistant/) showed something he had been working on to use the OpenAI integration along with Music Assistant. We now have this feature baked in to the integration code directly, although some extra setup is still required.
- You need to create/add another OpenAI integration that is purely for Music Assistant.
- Add the prompt found [here](https://github.com/jozefKruszynski/home-assistant-things/blob/main/blueprints/modified_prompt.txt) to the configuration of the the OpenAI integration.
- Add a directory in your Home Assistant `config` dir name `custom_sentences/en`
- Add the file found [here](https://github.com/music-assistant/hass-music-assistant/blob/main/custom_sentences/en/play_media_on_media_player.yaml), to that dir.
- When setting up the Music Assistant integration, make sure that you select the correct Conversation Agent and also
allow the auto-exposure of Mass media players to Assist

![Preview image](https://raw.githubusercontent.com/music-assistant/hass-music-assistant/main/screenshots/screen6.png)

## Usage and notes

- Music from your music sources will be automatically loaded into the Music Assistant library. If you have multiple sources, they will be merged as one library.
Expand Down
48 changes: 45 additions & 3 deletions custom_components/mass/config_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@
from homeassistant.core import HomeAssistant
from homeassistant.data_entry_flow import AbortFlow, FlowResult
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers import aiohttp_client
from homeassistant.helpers import aiohttp_client, selector
from music_assistant.client import MusicAssistantClient
from music_assistant.client.exceptions import CannotConnect, InvalidServerVersion
from music_assistant.common.models.api import ServerInfoMessage

from .addon import get_addon_manager, install_repository
from .const import (
ADDON_HOSTNAME,
CONF_ASSIST_AUTO_EXPOSE_PLAYERS,
CONF_INTEGRATION_CREATED_ADDON,
CONF_OPENAI_AGENT_ID,
CONF_USE_ADDON,
DOMAIN,
LOGGER,
Expand All @@ -37,13 +39,42 @@
DEFAULT_URL = "http://mass.local:8095"
ADDON_URL = f"http://{ADDON_HOSTNAME}:8095"
DEFAULT_TITLE = "Music Assistant"
ON_SUPERVISOR_SCHEMA = vol.Schema({vol.Optional(CONF_USE_ADDON, default=True): bool})

ON_SUPERVISOR_SCHEMA = vol.Schema(
{
vol.Optional(CONF_USE_ADDON, default=True): bool,
vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
selector.ConversationAgentSelectorConfig(language="en")
),
vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
}
)


def get_manual_schema(user_input: dict[str, Any]) -> vol.Schema:
"""Return a schema for the manual step."""
default_url = user_input.get(CONF_URL, DEFAULT_URL)
return vol.Schema({vol.Required(CONF_URL, default=default_url): str})
return vol.Schema(
{
vol.Required(CONF_URL, default=default_url): str,
vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
selector.ConversationAgentSelectorConfig(language="en")
),
vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
}
)


def get_zeroconf_schema() -> vol.Schema:
"""Return a schema for the zeroconf step."""
return vol.Schema(
{
vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
selector.ConversationAgentSelectorConfig(language="en")
),
vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
}
)


async def get_server_info(hass: HomeAssistant, url: str) -> ServerInfoMessage:
Expand All @@ -63,6 +94,8 @@ class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN):
def __init__(self) -> None:
"""Set up flow instance."""
self.server_info: ServerInfoMessage | None = None
self.openai_agent_id: str | None = None
self.expose_players_assist: bool | None = None
# If we install the add-on we should uninstall it on entry remove.
self.integration_created_addon = False
self.install_task: asyncio.Task | None = None
Expand Down Expand Up @@ -182,6 +215,8 @@ async def async_step_manual(self, user_input: dict[str, Any] | None = None) -> F

try:
self.server_info = await get_server_info(self.hass, user_input[CONF_URL])
self.openai_agent_id = user_input[CONF_OPENAI_AGENT_ID]
self.expose_players_assist = user_input[CONF_ASSIST_AUTO_EXPOSE_PLAYERS]
await self.async_set_unique_id(self.server_info.server_id)
except CannotConnect:
errors["base"] = "cannot_connect"
Expand Down Expand Up @@ -223,12 +258,15 @@ async def async_step_discovery_confirm(
if user_input is not None:
# Check that we can connect to the address.
try:
self.openai_agent_id = user_input[CONF_OPENAI_AGENT_ID]
self.expose_players_assist = user_input[CONF_ASSIST_AUTO_EXPOSE_PLAYERS]
await get_server_info(self.hass, self.server_info.base_url)
except CannotConnect:
return self.async_abort(reason="cannot_connect")
return await self._async_create_entry_or_abort()
return self.async_show_form(
step_id="discovery_confirm",
data_schema=get_zeroconf_schema(),
description_placeholders={"url": self.server_info.base_url},
)

Expand Down Expand Up @@ -279,6 +317,8 @@ async def _async_create_entry_or_abort(self) -> FlowResult:
CONF_URL: self.server_info.base_url,
CONF_USE_ADDON: self.use_addon,
CONF_INTEGRATION_CREATED_ADDON: self.integration_created_addon,
CONF_OPENAI_AGENT_ID: self.openai_agent_id,
CONF_ASSIST_AUTO_EXPOSE_PLAYERS: self.expose_players_assist,
},
title=DEFAULT_TITLE,
)
Expand All @@ -295,6 +335,8 @@ async def _async_create_entry_or_abort(self) -> FlowResult:
CONF_URL: self.server_info.base_url,
CONF_USE_ADDON: self.use_addon,
CONF_INTEGRATION_CREATED_ADDON: self.integration_created_addon,
CONF_OPENAI_AGENT_ID: self.openai_agent_id,
CONF_ASSIST_AUTO_EXPOSE_PLAYERS: self.expose_players_assist,
},
)

Expand Down
2 changes: 2 additions & 0 deletions custom_components/mass/const.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,7 @@

CONF_INTEGRATION_CREATED_ADDON = "integration_created_addon"
CONF_USE_ADDON = "use_addon"
CONF_OPENAI_AGENT_ID = "openai_agent_id"
CONF_ASSIST_AUTO_EXPOSE_PLAYERS = "expose_players_assist"

LOGGER = logging.getLogger(__package__)
205 changes: 205 additions & 0 deletions custom_components/mass/intent.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
"""Intents for the client integration."""
from __future__ import annotations

import json
from typing import TYPE_CHECKING, Any

import voluptuous as vol
from homeassistant.components.conversation import ATTR_AGENT_ID, ATTR_TEXT
from homeassistant.components.conversation import SERVICE_PROCESS as CONVERSATION_SERVICE
from homeassistant.components.conversation.const import DOMAIN as CONVERSATION_DOMAIN
from homeassistant.components.media_player.const import DOMAIN as MEDIA_PLAYER_DOMAIN
from homeassistant.config_entries import ConfigEntry, ConfigEntryState
from homeassistant.core import HomeAssistant
from homeassistant.helpers import area_registry as ar
from homeassistant.helpers import config_validation as cv
from homeassistant.helpers import device_registry as dr
from homeassistant.helpers import entity_registry as er
from homeassistant.helpers import intent

from . import DOMAIN
from .const import CONF_OPENAI_AGENT_ID
from .media_player import ATTR_MEDIA_ID, ATTR_MEDIA_TYPE, ATTR_RADIO_MODE, MassPlayer

if TYPE_CHECKING:
pass
if TYPE_CHECKING:
from music_assistant.client import MusicAssistantClient


INTENT_PLAY_MEDIA_ON_MEDIA_PLAYER = "MassPlayMediaOnMediaPlayer"
NAME_SLOT = "name"
AREA_SLOT = "area"
QUERY_SLOT = "query"
SLOT_VALUE = "value"


async def async_setup_intents(hass: HomeAssistant) -> None:
"""Set up the climate intents."""
intent.async_register(hass, MassPlayMediaOnMediaPlayerHandler())


class MassPlayMediaOnMediaPlayerHandler(intent.IntentHandler):
"""Handle PlayMediaOnMediaPlayer intents."""

intent_type = INTENT_PLAY_MEDIA_ON_MEDIA_PLAYER
slot_schema = {vol.Any(NAME_SLOT, AREA_SLOT): cv.string, vol.Optional(QUERY_SLOT): cv.string}

async def async_handle(self, intent_obj: intent.Intent) -> intent.IntentResponse:
"""Handle the intent."""
slots = self.async_validate_slots(intent_obj.slots)
config_entry = await self._get_loaded_config_entry(intent_obj.hass)

query: str = slots.get(QUERY_SLOT, {}).get(SLOT_VALUE)
if query is not None:
service_data: dict[str, Any] = {}
service_data[ATTR_AGENT_ID] = config_entry.data.get(CONF_OPENAI_AGENT_ID)
service_data[ATTR_TEXT] = query

# Look up area to fail early
area_name = slots.get(AREA_SLOT, {}).get(SLOT_VALUE)
if area_name is not None:
areas = ar.async_get(intent_obj.hass)
area_name = area_name.casefold()
area = await self._find_area(area_name, areas)
if area is None:
raise intent.IntentHandleError(f"No area named {area_name}")
media_player_entity = await self._get_entity_by_area(
area, intent_obj.hass, config_entry
)
if media_player_entity is None:
raise intent.IntentHandleError(f"No media player found matching area: {area_name}")

# Look up name to fail early
name: str = slots.get(NAME_SLOT, {}).get(SLOT_VALUE)
if name is not None:
name = name.casefold()
media_player_entity = await self._get_entity_from_registry(
name, intent_obj.hass, config_entry
)
if media_player_entity is None:
raise intent.IntentHandleError(f"No media player found matching name: {name}")

actual_player = await self._get_mass_player_from_registry_entry(
intent_obj.hass, config_entry, media_player_entity
)
if actual_player is None:
raise intent.IntentHandleError(f"No Mass media player found for name {name}")

media_id, media_type = await self._get_media_id_and_media_type_from_query_result(
intent_obj.hass, service_data, intent_obj
)
await actual_player.async_play_media(
media_id=media_id, media_type=media_type, extra={ATTR_RADIO_MODE: False}
)
response = intent_obj.create_response()
response.response_type = intent.IntentResponseType.ACTION_DONE
if area_name is not None:
response.async_set_speech(f"Playing selection in {area_name}")
if name is not None:
response.async_set_speech(f"Playing selection on {name}")
return response

async def _get_media_id_and_media_type_from_query_result(
self, hass: HomeAssistant, service_data: dict[str, Any], intent_obj: intent.Intent
) -> str:
"""Get from the query."""
ai_response = await hass.services.async_call(
CONVERSATION_DOMAIN,
CONVERSATION_SERVICE,
{**service_data},
blocking=True,
context=intent_obj.context,
return_response=True,
)
json_payload = json.loads(ai_response["response"]["speech"]["plain"]["speech"])
media_id = json_payload.get(ATTR_MEDIA_ID)
media_type = json_payload.get(ATTR_MEDIA_TYPE)
return media_id, media_type

async def _get_loaded_config_entry(self, hass: HomeAssistant) -> str:
"""Get the correct config entry."""
config_entries = hass.config_entries.async_entries(DOMAIN)
for config_entry in config_entries:
if config_entry.state == ConfigEntryState.LOADED:
return config_entry
return None

async def _get_entity_from_registry(
self, name: str, hass: HomeAssistant, config_entry: ConfigEntry
) -> er.RegistryEntry:
"""Get the entity from the registry."""
entity_registry = er.async_get(hass)
entity_registry_entries = er.async_entries_for_config_entry(
entity_registry, config_entry.entry_id
)
for entity_registry_entry in entity_registry_entries:
if await self._has_name(entity_registry_entry, name):
return entity_registry_entry
return None

async def _has_name(self, entity: er.RegistryEntry | None, name: str) -> bool:
"""Return true if entity name or alias matches."""
if entity is not None:
normalised_entity_id = (
entity.entity_id.replace("_", " ").strip("media player.").casefold()
)

if name in normalised_entity_id:
return True

# Check name/aliases
if (entity is None) or (not entity.aliases):
return False

return any(name == alias.casefold() for alias in entity.aliases)

async def _get_mass_player_from_registry_entry(
self, hass: HomeAssistant, config_entry: ConfigEntry, media_player_entity: er.RegistryEntry
) -> MassPlayer:
"""Return the mass player."""
mass: MusicAssistantClient = hass.data[DOMAIN][config_entry.entry_id].mass
player_entity = hass.data[MEDIA_PLAYER_DOMAIN].get_entity(media_player_entity.entity_id)
mass_player = mass.players.get_player(
player_entity.extra_state_attributes.get("mass_player_id")
)
actual_player = MassPlayer(mass, mass_player.player_id)
return actual_player

async def _find_area(self, area_name: str, areas: ar.AreaRegistry) -> ar.AreaEntry | None:
"""Find an area by id or name, checking aliases too."""
area = areas.async_get_area(area_name) or areas.async_get_area_by_name(area_name)
if area is not None:
return area

# Check area aliases
for maybe_area in areas.areas.values():
if not maybe_area.aliases:
continue

for area_alias in maybe_area.aliases:
if area_name == area_alias.casefold():
return maybe_area

return None

async def _get_entity_by_area(
self, area: ar.AreaEntry, hass: HomeAssistant, config_entry: ConfigEntry
) -> er.RegistryEntry:
"""Filter state/entity pairs by an area."""
entity_registry = er.async_get(hass)
device_registry = dr.async_get(hass)
entity_registry_entries = er.async_entries_for_config_entry(
entity_registry, config_entry.entry_id
)
for entity_registry_entry in entity_registry_entries:
if entity_registry_entry.area_id == area.id:
# Use entity's area id first
return entity_registry_entry
if entity_registry_entry.device_id is not None:
# Fall back to device area if not set on entity
device = device_registry.async_get(entity_registry_entry.device_id)
if device is not None and device.area_id == area.id:
return entity_registry_entry

return None
17 changes: 16 additions & 1 deletion custom_components/mass/media_player.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import homeassistant.helpers.config_validation as cv
import voluptuous as vol
from homeassistant.components import media_source
from homeassistant.components.homeassistant.exposed_entities import async_expose_entity
from homeassistant.components.media_player import (
BrowseMedia,
MediaPlayerDeviceClass,
Expand All @@ -22,7 +23,7 @@
ATTR_MEDIA_EXTRA,
MediaPlayerEntityFeature,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.config_entries import ConfigEntry, ConfigEntryState
from homeassistant.const import STATE_IDLE, STATE_OFF, STATE_PAUSED, STATE_PLAYING
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback, async_get_current_platform
Expand Down Expand Up @@ -172,6 +173,10 @@ def __init__(self, mass: MusicAssistantClient, player_id: str) -> None:
async def async_added_to_hass(self) -> None:
"""Register callbacks."""
await super().async_added_to_hass()
# we need to get the hass object in order to get our config entry
# and expose the player to the conversation component, assuming that
# the config entry has the option enabled.
await self._expose_players_assist()

# we subscribe to player queue time update but we only
# accept a state change on big time jumps (e.g. seeking)
Expand Down Expand Up @@ -553,3 +558,13 @@ async def _get_item_by_name(
# simply return the first item because search is already sorted by best match
return item
return None

async def _expose_players_assist(self) -> None:
"""Get the correct config entry."""
hass = self.hass
config_entries = hass.config_entries.async_entries(DOMAIN)
for config_entry in config_entries:
if config_entry.state == ConfigEntryState.SETUP_IN_PROGRESS and config_entry.data.get(
"expose_players_assist"
):
async_expose_entity(hass, "conversation", self.entity_id, True)
Loading

0 comments on commit ba0a34d

Please sign in to comment.