From efee9589fa99ac91ff83228520be42d17d7a405f Mon Sep 17 00:00:00 2001 From: Eric Gustin <34000337+EricGustin@users.noreply.github.com> Date: Fri, 1 Nov 2024 13:15:43 -0700 Subject: [PATCH] More Spotify Tools (#140) # PR Description This PR adds three new spotify tools that are natural language friendly. 1. `search` - Search Spotify Catalog information 2. `play_artist_by_name` - Gets 5 songs by the specified artist and plays them. Uses `search`, and `start_tracks_playback_by_id` under the hood 3. `play_track_by_name` - Plays the specified song, optionally provide the artist name who plays the song. Uses `search`, and `start_tracks_playback_by_id` under the hood --- .../spotify/arcade_spotify/tools/constants.py | 2 + .../spotify/arcade_spotify/tools/models.py | 27 ++++++ .../spotify/arcade_spotify/tools/player.py | 93 ++++++++++++++++++- .../spotify/arcade_spotify/tools/search.py | 39 ++++++++ toolkits/spotify/evals/eval_player.py | 41 +++++++- toolkits/spotify/evals/eval_search.py | 52 +++++++++++ toolkits/spotify/evals/eval_tracks.py | 2 +- 7 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 toolkits/spotify/arcade_spotify/tools/search.py create mode 100644 toolkits/spotify/evals/eval_search.py diff --git a/toolkits/spotify/arcade_spotify/tools/constants.py b/toolkits/spotify/arcade_spotify/tools/constants.py index ff8bf38b..9b53326d 100644 --- a/toolkits/spotify/arcade_spotify/tools/constants.py +++ b/toolkits/spotify/arcade_spotify/tools/constants.py @@ -1,6 +1,7 @@ SPOTIFY_BASE_URL = "https://api.spotify.com/v1" ENDPOINTS = { + "player_get_available_devices": "/me/player/devices", "player_get_playback_state": "/me/player", "player_get_currently_playing": "/me/player/currently-playing", "player_modify_playback": "/me/player/play", @@ -11,4 +12,5 @@ ENDPOINTS = { "tracks_get_track": "/tracks/{track_id}", "tracks_get_recommendations": "/recommendations", "tracks_get_audio_features": "/audio-features", + "search": "/search", } diff --git a/toolkits/spotify/arcade_spotify/tools/models.py b/toolkits/spotify/arcade_spotify/tools/models.py index 9f625f32..fe0f6382 100644 --- a/toolkits/spotify/arcade_spotify/tools/models.py +++ b/toolkits/spotify/arcade_spotify/tools/models.py @@ -1,4 +1,5 @@ from dataclasses import asdict, dataclass, field +from enum import Enum from typing import Optional @@ -27,3 +28,29 @@ class PlaybackState: def to_dict(self) -> dict: """Convert the PlaybackState instance to a dictionary, excluding None values.""" return {k: v for k, v in asdict(self).items() if v is not None and v != []} + + +@dataclass +class Device: + id: str + is_active: bool + is_private_session: bool + is_restricted: bool + name: str + type: str + volume_percent: int + supports_volume: bool + + def to_dict(self) -> dict: + """Convert the Device instance to a dictionary.""" + return asdict(self) + + +class SearchType(str, Enum): + ALBUM = "album" + ARTIST = "artist" + PLAYLIST = "playlist" + TRACK = "track" + SHOW = "show" + EPISODE = "episode" + AUDIOBOOK = "audiobook" diff --git a/toolkits/spotify/arcade_spotify/tools/player.py b/toolkits/spotify/arcade_spotify/tools/player.py index 8fdb6f7e..f4938747 100644 --- a/toolkits/spotify/arcade_spotify/tools/player.py +++ b/toolkits/spotify/arcade_spotify/tools/player.py @@ -3,6 +3,8 @@ from typing import Annotated, Optional from arcade.sdk import ToolContext, tool from arcade.sdk.auth import Spotify from arcade.sdk.errors import RetryableToolError +from arcade_spotify.tools.models import Device, SearchType +from arcade_spotify.tools.search import search from arcade_spotify.tools.utils import ( convert_to_playback_state, get_url, @@ -183,13 +185,31 @@ async def start_tracks_playback_by_id( ] = 0, ) -> Annotated[dict, "The updated playback state"]: """Start playback of a list of tracks (songs)""" + + devices = [ + Device(**device) for device in (await get_available_devices(context)).get("devices", []) + ] + + # If no active device is available, pick the first one. Otherwise, Spotify defaults to the active device. + device_id = None + if devices and not any(device.is_active for device in devices): + device_id = devices[0].id + + params = {"device_id": device_id} if device_id else {} + url = get_url("player_modify_playback") body = { "uris": [f"spotify:track:{track_id}" for track_id in track_ids], "position_ms": position_ms, } - response = await send_spotify_request(context, "PUT", url, json_data=body) + response = await send_spotify_request(context, "PUT", url, params=params, json_data=body) + playback_state = handle_404_playback_state( + response, "Cannot start playback because no active device is available", False + ) + if playback_state: + return playback_state + response.raise_for_status() playback_state = await get_playback_state(context) @@ -221,3 +241,74 @@ async def get_currently_playing( response.raise_for_status() data = {"is_playing": False} if response.status_code == 204 else response.json() return convert_to_playback_state(data).to_dict() + + +# NOTE: This tool only works for Spotify Premium users +@tool( + requires_auth=Spotify( + scopes=["user-read-playback-state", "user-modify-playback-state"], + ) +) +async def play_artist_by_name( + context: ToolContext, + name: Annotated[str, "The name of the artist to play"], +) -> Annotated[dict, "The updated playback state"]: + """Plays a song by an artist and queues four more songs by the same artist""" + q = f"artist:{name}" + search_results = await search(context, q, [SearchType.TRACK], 5) + if not search_results["tracks"]["items"]: + message = f"Artist '{name}' not found." + raise RetryableToolError( + message, + additional_prompt_content=f"{message} Try a different artist name.", + retry_after_ms=500, + ) + track_ids = [item["id"] for item in search_results["tracks"]["items"]] + playback_state = await start_tracks_playback_by_id(context, track_ids) + + return playback_state + + +# NOTE: This tool only works for Spotify Premium users +@tool( + requires_auth=Spotify( + scopes=["user-read-playback-state", "user-modify-playback-state"], + ) +) +async def play_track_by_name( + context: ToolContext, + track_name: Annotated[str, "The name of the track to play"], + artist_name: Annotated[Optional[str], "The name of the artist of the track"] = None, +) -> Annotated[dict, "The updated playback state"]: + """Plays a song by name""" + q = f"track:{track_name}" + if artist_name: + q += f" artist:{artist_name}" + search_results = await search(context, q, [SearchType.TRACK], 1) + + if not search_results["tracks"]["items"]: + message = f"No track exists with name '{track_name}'" + if artist_name: + message += f" by '{artist_name}'" + raise RetryableToolError( + message, + additional_prompt_content=f"{message}. Try a different track name or artist name.", + retry_after_ms=500, + ) + + track_id = search_results["tracks"]["items"][0]["id"] + playback_state = await start_tracks_playback_by_id(context, [track_id]) + + return playback_state + + +# NOTE: This tool only works for Spotify Premium users +@tool(requires_auth=Spotify(scopes=["user-read-playback-state"])) +async def get_available_devices( + context: ToolContext, +) -> Annotated[dict, "The available devices"]: + """Get the available devices""" + url = get_url("player_get_available_devices") + response = await send_spotify_request(context, "GET", url) + response.raise_for_status() + return response.json() diff --git a/toolkits/spotify/arcade_spotify/tools/search.py b/toolkits/spotify/arcade_spotify/tools/search.py new file mode 100644 index 00000000..bddb7b33 --- /dev/null +++ b/toolkits/spotify/arcade_spotify/tools/search.py @@ -0,0 +1,39 @@ +from typing import Annotated + +from arcade.sdk import ToolContext, tool +from arcade.sdk.auth import Spotify +from arcade_spotify.tools.models import SearchType +from arcade_spotify.tools.utils import ( + get_url, + send_spotify_request, +) + + +@tool(requires_auth=Spotify()) +async def search( + context: ToolContext, + q: Annotated[str, "The search query"], + types: Annotated[list[SearchType], "The types of results to return"], + limit: Annotated[int, "The maximum number of results to return"] = 1, +) -> Annotated[dict, "A list of artists matching the search query"]: + """Search Spotify catalog information + + Explanation of the q parameter: + You can narrow down your search using field filters. The available filters are album, artist, track, year, upc, tag:hipster, tag:new, isrc, and genre. Each field filter only applies to certain result types. + + The artist and year filters can be used while searching albums, artists and tracks. You can filter on a single year or a range (e.g. 1955-1960). + The album filter can be used while searching albums and tracks. + The genre filter can be used while searching artists and tracks. + The isrc and track filters can be used while searching tracks. + The upc, tag:new and tag:hipster filters can only be used while searching albums. The tag:new filter will return albums released in the past two weeks and tag:hipster can be used to return only albums with the lowest 10% popularity. + + Example: q="remaster track:Doxy artist:Miles Davis" + """ + + url = get_url("search", q=q) + + response = await send_spotify_request( + context, "GET", url, params={"q": q, "type": ",".join(types), "limit": limit} + ) + response.raise_for_status() + return response.json() diff --git a/toolkits/spotify/evals/eval_player.py b/toolkits/spotify/evals/eval_player.py index 8314d940..3513bb78 100644 --- a/toolkits/spotify/evals/eval_player.py +++ b/toolkits/spotify/evals/eval_player.py @@ -3,6 +3,8 @@ from arcade_spotify.tools.player import ( get_currently_playing, get_playback_state, pause_playback, + play_artist_by_name, + play_track_by_name, resume_playback, skip_to_next_track, skip_to_previous_track, @@ -15,7 +17,7 @@ from arcade.sdk.eval import ( EvalSuite, tool_eval, ) -from arcade.sdk.eval.critic import NumericCritic +from arcade.sdk.eval.critic import BinaryCritic, NumericCritic, SimilarityCritic # Evaluation rubric rubric = EvalRubric( @@ -32,10 +34,12 @@ catalog.add_tool(resume_playback, "Spotify") catalog.add_tool(start_tracks_playback_by_id, "Spotify") catalog.add_tool(get_playback_state, "Spotify") catalog.add_tool(get_currently_playing, "Spotify") +catalog.add_tool(play_artist_by_name, "Spotify") +catalog.add_tool(play_track_by_name, "Spotify") @tool_eval() -def spotify_eval_suite() -> EvalSuite: +def spotify_player_eval_suite() -> EvalSuite: """Create an evaluation suite for Spotify "player" tools.""" suite = EvalSuite( name="Spotify Tools Evaluation", @@ -118,6 +122,10 @@ def spotify_eval_suite() -> EvalSuite: }, ) ], + critics=[ + BinaryCritic(critic_field="track_ids", weight=0.5), + NumericCritic(critic_field="position_ms", weight=0.5, value_range=(9000, 11000)), + ], ) suite.add_case( @@ -132,4 +140,33 @@ def spotify_eval_suite() -> EvalSuite: expected_tool_calls=[(get_playback_state, {})], ) + suite.add_case( + name="Play artist by name", + user_message="play pearl jam", + expected_tool_calls=[ + ( + play_artist_by_name, + {"name": "Pearl Jam"}, + ) + ], + critics=[ + SimilarityCritic(critic_field="name", weight=1.0), + ], + ) + + suite.add_case( + name="Play track by name", + user_message="it would be really great if I could listen to strobe by deadmau5 right now.", + expected_tool_calls=[ + ( + play_track_by_name, + {"track_name": "strobe", "artist_name": "deadmau5"}, + ) + ], + critics=[ + SimilarityCritic(critic_field="track_name", weight=0.5), + SimilarityCritic(critic_field="artist_name", weight=0.5), + ], + ) + return suite diff --git a/toolkits/spotify/evals/eval_search.py b/toolkits/spotify/evals/eval_search.py new file mode 100644 index 00000000..f8253222 --- /dev/null +++ b/toolkits/spotify/evals/eval_search.py @@ -0,0 +1,52 @@ +from arcade_spotify.tools.models import SearchType +from arcade_spotify.tools.search import search + +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + EvalRubric, + EvalSuite, + tool_eval, +) +from arcade.sdk.eval.critic import BinaryCritic, SimilarityCritic + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + +catalog = ToolCatalog() +catalog.add_tool(search, "Spotify") + + +@tool_eval() +def spotify_search_eval_suite() -> EvalSuite: + """Create an evaluation suite for Spotify "player" tools.""" + suite = EvalSuite( + name="Spotify Tools Evaluation", + system_message="You are an AI assistant that can manage Spotify using the provided tools.", + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Search Spotify catalog", + user_message="search for 3 songs in the the album 'American IV: The Man Comes Around' by Johnny Cash", + expected_tool_calls=[ + ( + search, + { + "q": "album:American IV: The Man Comes Around artist:Johnny Cash", + "types": [SearchType.TRACK], + "limit": 3, + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="q", weight=0.5), + BinaryCritic(critic_field="limit", weight=0.25), + BinaryCritic(critic_field="types", weight=0.25), + ], + ) + + return suite diff --git a/toolkits/spotify/evals/eval_tracks.py b/toolkits/spotify/evals/eval_tracks.py index 87e7558a..97e74650 100644 --- a/toolkits/spotify/evals/eval_tracks.py +++ b/toolkits/spotify/evals/eval_tracks.py @@ -130,7 +130,7 @@ history_3 = [ @tool_eval() -def spotify_eval_suite() -> EvalSuite: +def spotify_tracks_eval_suite() -> EvalSuite: """Create an evaluation suite for Spotify "track" tools.""" suite = EvalSuite( name="Spotify Tools Evaluation",