diff --git a/TODO b/TODO index 7e4e7b2..a5ae0d8 100644 --- a/TODO +++ b/TODO @@ -5,13 +5,13 @@ - new release for amazon tv series - allow amazon tv search for indivdual series - new release for cinema-paradiso tv / movie -- allow setting of amazon region in settings -- allow setting of musicbrainz url in settings ## bugs - music, a-ha/ash doesnt match as an artist why ? - when scraping movies, do we stop at the first best match ? +- speed up plex fetch of movie details +- speed up plex fetch of tv shows ## done @@ -40,3 +40,5 @@ - parallelise amazon search tv/movie - move newer show out of amazon and cinema-paradiso, move to web page - move language filtering out of plex search, should only happen in web tv & movie web pages +- allow setting of amazon region in settings +- allow setting of musicbrainz url in settings diff --git a/amazon/amazon.go b/amazon/amazon.go index 33ed83e..f316d1c 100644 --- a/amazon/amazon.go +++ b/amazon/amazon.go @@ -26,7 +26,7 @@ var ( ) // nolint: dupl, nolintlint -func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string) (searchResults []types.SearchResults) { +func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language, region string) (searchResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(plexMovies)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -35,7 +35,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string) go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - searchAmazonMovie(&plexMovies[i], language, ch) + searchAmazonMovie(&plexMovies[i], language, region, ch) }(i) } @@ -51,7 +51,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string) } // nolint: dupl, nolintlint -func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language string) (searchResults []types.SearchResults) { +func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language, region string) (searchResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(plexTVShows)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -60,7 +60,7 @@ func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language string) ( go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - searchAmazonTV(&plexTVShows[i], language, ch) + searchAmazonTV(&plexTVShows[i], language, region, ch) }(i) } @@ -83,7 +83,7 @@ func GetTVJobProgress() int { return numberTVProcessed } -func ScrapeTitlesParallel(searchResults []types.SearchResults) (scrapedResults []types.SearchResults) { +func ScrapeTitlesParallel(searchResults []types.SearchResults, region string) (scrapedResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(searchResults)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -91,7 +91,7 @@ func ScrapeTitlesParallel(searchResults []types.SearchResults) (scrapedResults [ go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - scrapeTitles(&searchResults[i], ch) + scrapeTitles(&searchResults[i], region, ch) }(i) } @@ -106,14 +106,14 @@ func ScrapeTitlesParallel(searchResults []types.SearchResults) (scrapedResults [ return scrapedResults } -func scrapeTitles(searchResult *types.SearchResults, ch chan<- types.SearchResults) { +func scrapeTitles(searchResult *types.SearchResults, region string, ch chan<- types.SearchResults) { dateAdded := searchResult.PlexMovie.DateAdded for i := range searchResult.MovieSearchResults { // this is to limit the number of requests if !searchResult.MovieSearchResults[i].BestMatch { continue } - rawData, err := makeRequest(searchResult.MovieSearchResults[i].URL, "") + rawData, err := makeRequest(searchResult.MovieSearchResults[i].URL, region) if err != nil { fmt.Println("scrapeTitle: Error making request:", err) ch <- *searchResult @@ -134,7 +134,7 @@ func scrapeTitles(searchResult *types.SearchResults, ch chan<- types.SearchResul ch <- *searchResult } -func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchResult chan<- types.SearchResults) { +func searchAmazonMovie(plexMovie *types.PlexMovie, language, region string, movieSearchResult chan<- types.SearchResults) { result := types.SearchResults{} result.PlexMovie = *plexMovie result.SearchURL = "" @@ -150,7 +150,7 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchR } amazonURL += "&submit=Search&action=search" - rawData, err := makeRequest(amazonURL, language) + rawData, err := makeRequest(amazonURL, region) if err != nil { fmt.Println("searchAmazonMovie: Error making request:", err) movieSearchResult <- result @@ -163,7 +163,7 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchR movieSearchResult <- result } -func searchAmazonTV(plexTVShow *types.PlexTVShow, language string, tvSearchResult chan<- types.SearchResults) { +func searchAmazonTV(plexTVShow *types.PlexTVShow, language, region string, tvSearchResult chan<- types.SearchResults) { result := types.SearchResults{} result.PlexTVShow = *plexTVShow result.SearchURL = amazonURL @@ -179,7 +179,7 @@ func searchAmazonTV(plexTVShow *types.PlexTVShow, language string, tvSearchResul } amazonURL += "&submit=Search&action=search" - rawData, err := makeRequest(amazonURL, language) + rawData, err := makeRequest(amazonURL, region) if err != nil { fmt.Println("searchAmazonTV: Error making request:", err) tvSearchResult <- result @@ -254,19 +254,13 @@ func findTitlesInResponse(response string, movie bool) (movieResults []types.Mov return movieResults, tvResults } -func makeRequest(inputURL, language string) (response string, err error) { +func makeRequest(inputURL, region string) (response string, err error) { req, err := http.NewRequestWithContext(context.Background(), "GET", inputURL, bytes.NewBuffer([]byte{})) req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3") - // this forces results from a specific amazon region - switch language { - case LanguageGerman: - req.Header.Set("Cookie", "country=de;") - default: - req.Header.Set("Cookie", "country=uk;") - } + req.Header.Set("Cookie", fmt.Sprintf("country=%s;", region)) if err != nil { fmt.Println("makeRequest: error creating request:", err) diff --git a/amazon/amazon_test.go b/amazon/amazon_test.go index a661f50..48bcada 100644 --- a/amazon/amazon_test.go +++ b/amazon/amazon_test.go @@ -8,6 +8,10 @@ import ( "github.com/tphoney/plex-lookup/types" ) +const ( + amazonRegion = "uk" +) + var ( plexIP = os.Getenv("PLEX_IP") plexToken = os.Getenv("PLEX_TOKEN") @@ -39,7 +43,7 @@ func TestFindMoviesInResponse(t *testing.T) { } func TestSearchAmazon(t *testing.T) { - result := SearchAmazonMoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "") + result := SearchAmazonMoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "", amazonRegion) if len(result) == 0 { t.Errorf("Expected search results, but got none") } @@ -58,7 +62,7 @@ func TestSearchAmazonTV(t *testing.T) { // Title: "Adventure Time", // Year: "2010", } - result := SearchAmazonTVInParallel([]types.PlexTVShow{show}, "") + result := SearchAmazonTVInParallel([]types.PlexTVShow{show}, "", amazonRegion) if len(result) == 0 { t.Errorf("Expected search results, but got none") @@ -81,7 +85,7 @@ func TestScrapeTitlesParallel(t *testing.T) { }, }, }, - }) + }, amazonRegion) if len(result) == 0 { t.Errorf("Expected search results, but got none") diff --git a/cmd/amazon.go b/cmd/amazon.go index b3a11f2..59e8e1b 100644 --- a/cmd/amazon.go +++ b/cmd/amazon.go @@ -24,7 +24,7 @@ func performAmazonLookup() { if libraryType == types.PlexMovieType { plexMovies := initializePlexMovies() // lets search movies in amazon - searchResults := amazon.SearchAmazonMoviesInParallel(plexMovies, "") + searchResults := amazon.SearchAmazonMoviesInParallel(plexMovies, "", amazonRegion) for i := range searchResults { for _, individualResult := range searchResults[i].MovieSearchResults { if individualResult.BestMatch && (individualResult.Format == types.DiskBluray || individualResult.Format == types.Disk4K) { diff --git a/cmd/root.go b/cmd/root.go index 0e5f5c4..97142f4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,10 @@ import ( "github.com/tphoney/plex-lookup/types" ) +const ( + amazonRegion = "uk" +) + var ( // Used for flags. plexIP string diff --git a/cmd/web.go b/cmd/web.go index 6cc2f1b..61f1dcc 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -25,6 +25,14 @@ func startServer() { config.PlexTVLibraryID = os.Getenv("PLEX_TV_LIBRARY_ID") config.PlexMusicLibraryID = os.Getenv("PLEX_MUSIC_LIBRARY_ID") config.PlexToken = os.Getenv("PLEX_TOKEN") + config.AmazonRegion = os.Getenv("AMAZON_REGION") + if config.AmazonRegion == "" { + config.AmazonRegion = "uk" + } + config.MusicBrainzURL = os.Getenv("MUSICBRAINZ_URL") + if config.MusicBrainzURL == "" { + config.MusicBrainzURL = "https://musicbrainz.org/ws/2" + } config.SpotifyClientID = os.Getenv("SPOTIFY_CLIENT_ID") config.SpotifyClientSecret = os.Getenv("SPOTIFY_CLIENT_SECRET") diff --git a/musicbrainz/musicbrainz.go b/musicbrainz/musicbrainz.go index 09d14ca..ec84282 100644 --- a/musicbrainz/musicbrainz.go +++ b/musicbrainz/musicbrainz.go @@ -14,15 +14,13 @@ import ( // example artist https://musicbrainz.org/artist/83d91898-7763-47d7-b03b-b92132375c47 const ( - // MusicBrainzURL is the URL for the MusicBrainz API - musicBrainzURL = "https://musicbrainz.org/ws/2" - agent = "plex-lookup" - agentVersion = "0.0.1" - lookupLimit = 100 - lookupTimeout = 2 + agent = "plex-lookup" + agentVersion = "0.0.1" + lookupLimit = 100 + lookupTimeout = 2 ) -func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.SearchResults, err error) { +func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist, musicBrainzURL string) (artist types.SearchResults, err error) { artist.PlexMusicArtist = *plexArtist client, err := gomusicbrainz.NewWS2Client( musicBrainzURL, agent, agentVersion, "") @@ -58,9 +56,9 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se if err != nil { // check for a 503 error if err.Error() == "EOF" { - fmt.Println("SearchMusicBrainzArtist rate limit exceeded") + fmt.Printf("!") time.Sleep(lookupTimeout * time.Second) - return SearchMusicBrainzArtist(plexArtist) + return SearchMusicBrainzArtist(plexArtist, musicBrainzURL) } } @@ -75,7 +73,7 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se url := fmt.Sprintf("https://musicbrainz.org/artist/%v", found.ID) found.URL = url // get the albums - found.Albums, _ = SearchMusicBrainzAlbums(found.ID) + found.Albums, _ = SearchMusicBrainzAlbums(found.ID, musicBrainzURL) artist.MusicSearchResults = append(artist.MusicSearchResults, found) break } @@ -85,7 +83,7 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se return artist, err } -func SearchMusicBrainzAlbums(artistID string) (albums []types.MusicAlbumSearchResult, err error) { +func SearchMusicBrainzAlbums(artistID, musicBrainzURL string) (albums []types.MusicAlbumSearchResult, err error) { client, err := gomusicbrainz.NewWS2Client( musicBrainzURL, agent, agentVersion, "") @@ -98,9 +96,9 @@ func SearchMusicBrainzAlbums(artistID string) (albums []types.MusicAlbumSearchRe resp, err := client.SearchReleaseGroup(queryURL, lookupLimit, -1) if err != nil { if err.Error() == "EOF" { - fmt.Println("SearchMusicBrainzAlbums rate limit exceeded") + fmt.Printf("!") time.Sleep(lookupTimeout * time.Second) - return SearchMusicBrainzAlbums(artistID) + return SearchMusicBrainzAlbums(artistID, musicBrainzURL) } } for i := range resp.ReleaseGroups { diff --git a/musicbrainz/musicbrainz_test.go b/musicbrainz/musicbrainz_test.go index 89867f6..895d7c6 100644 --- a/musicbrainz/musicbrainz_test.go +++ b/musicbrainz/musicbrainz_test.go @@ -6,6 +6,10 @@ import ( "github.com/tphoney/plex-lookup/types" ) +const ( + musicBrainzURL = "https://musicbrainz.org/ws/2" +) + func TestSearchMusicBrainzArtist(t *testing.T) { tests := []struct { name string @@ -47,7 +51,7 @@ func TestSearchMusicBrainzArtist(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotArtist, err := SearchMusicBrainzArtist(tt.args) + gotArtist, err := SearchMusicBrainzArtist(tt.args, musicBrainzURL) if (err != nil) != tt.wantErr { t.Errorf("SearchMusicBrainzArtist() error = %v, wantErr %v", err, tt.wantErr) return @@ -66,7 +70,7 @@ func TestSearchMusicBrainzArtist(t *testing.T) { // debug test for individual artists func TestSearchMusicBrainzArtistDebug(t *testing.T) { artist := &types.PlexMusicArtist{Name: "Aaliyah"} - artistSearchResult, err := SearchMusicBrainzArtist(artist) + artistSearchResult, err := SearchMusicBrainzArtist(artist, musicBrainzURL) if err != nil { t.Errorf("SearchMusicBrainzArtist() error = %v", err) } @@ -92,7 +96,7 @@ func TestSearchMusicBrainzAlbums(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotAlbums, err := SearchMusicBrainzAlbums(tt.args.artistID) + gotAlbums, err := SearchMusicBrainzAlbums(tt.args.artistID, musicBrainzURL) if (err != nil) != tt.wantErr { t.Errorf("SearchMusicBrainzAlbums() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/types/types.go b/types/types.go index c9327d3..e7dcab6 100644 --- a/types/types.go +++ b/types/types.go @@ -36,6 +36,8 @@ type Configuration struct { PlexMovieLibraryID string PlexTVLibraryID string PlexMusicLibraryID string + AmazonRegion string + MusicBrainzURL string SpotifyClientID string SpotifyClientSecret string } diff --git a/web/movies/movies.go b/web/movies/movies.go index 56d1f84..92e0bab 100644 --- a/web/movies/movies.go +++ b/web/movies/movies.go @@ -84,10 +84,10 @@ func (c MoviesConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { if lookup == "cinemaParadiso" { searchResults = cinemaparadiso.GetCinemaParadisoMoviesInParallel(filteredPlexMovies) } else { - searchResults = amazon.SearchAmazonMoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage) + searchResults = amazon.SearchAmazonMoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage, c.Config.AmazonRegion) // if we are filtering by newer version, we need to search again if lookupFilters.NewerVersion { - searchResults = amazon.ScrapeTitlesParallel(searchResults) + searchResults = amazon.ScrapeTitlesParallel(searchResults, c.Config.AmazonRegion) } } diff --git a/web/music/music.go b/web/music/music.go index 3664bb0..ae232d2 100644 --- a/web/music/music.go +++ b/web/music/music.go @@ -6,6 +6,7 @@ import ( "html/template" "net/http" "strconv" + "strings" "time" "github.com/tphoney/plex-lookup/musicbrainz" @@ -48,13 +49,25 @@ func MusicHandler(w http.ResponseWriter, _ *http.Request) { // nolint: lll, nolintlint func (c MusicConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { lookup = r.FormValue("lookup") + if lookup == "musicbrainz" { + if c.Config.MusicBrainzURL == "" { + fmt.Fprintf(w, `
MusicBrainz URL is not set. Please set in settings.
`) + return + } + } + if lookup == "spotify" { + if c.Config.SpotifyClientID == "" || c.Config.SpotifyClientSecret == "" { + fmt.Fprintf(w, `
Spotify Client ID or Secret is not set. Please set in settings.
`) + return + } + } lookupType = r.FormValue("lookuptype") // only get the artists from plex once if len(plexMusic) == 0 { plexMusic = plex.GetPlexMusicArtists(c.Config.PlexIP, c.Config.PlexMusicLibraryID, c.Config.PlexToken) } //nolint: gocritic - // plexMusic = plexMusic[:10] + // plexMusic = plexMusic[:30] //lint: gocritic var searchResult types.SearchResults artistsJobRunning = true @@ -67,13 +80,16 @@ func (c MusicConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { if lookupType == "missingalbums" { switch lookup { case "musicbrainz": - plexMusic = plexMusic[:50] - totalArtists = len(plexMusic) - 1 + // limit the number of artists to 50 for nonlocal musicbrainz instances + if strings.Contains(c.Config.MusicBrainzURL, "musicbrainz.org") { + plexMusic = plexMusic[:50] + totalArtists = len(plexMusic) - 1 + } go func() { startTime := time.Now() for i := range plexMusic { fmt.Print(".") - searchResult, _ = musicbrainz.SearchMusicBrainzArtist(&plexMusic[i]) + searchResult, _ = musicbrainz.SearchMusicBrainzArtist(&plexMusic[i], c.Config.MusicBrainzURL) artistsSearchResults = append(artistsSearchResults, searchResult) numberOfArtistsProcessed = i } diff --git a/web/server.go b/web/server.go index ee7c14e..ac68a8e 100644 --- a/web/server.go +++ b/web/server.go @@ -78,6 +78,8 @@ func settingsSaveHandler(w http.ResponseWriter, r *http.Request) { config.PlexMovieLibraryID = r.FormValue("plexMovieLibraryID") config.PlexTVLibraryID = r.FormValue("plexTVLibraryID") config.PlexMusicLibraryID = r.FormValue("plexMusicLibraryID") + config.AmazonRegion = r.FormValue("amazonRegion") + config.MusicBrainzURL = r.FormValue("musicBrainzURL") config.SpotifyClientID = r.FormValue("spotifyClientID") config.SpotifyClientSecret = r.FormValue("spotifyClientSecret") fmt.Fprint(w, `

Saved!

Back`) diff --git a/web/settings/settings.go b/web/settings/settings.go index b960c36..37fc329 100644 --- a/web/settings/settings.go +++ b/web/settings/settings.go @@ -47,7 +47,6 @@ func ProcessPlexLibrariesHTML(w http.ResponseWriter, r *http.Request) { func (c SettingsConfig) PlexInformationOKHTML(w http.ResponseWriter, r *http.Request) { currentURL := r.Header.Get("hx-current-url") // get the last part of the url - requestingPage := path.Base(currentURL) if c.Config.PlexIP == "" || c.Config.PlexToken == "" { fmt.Fprint(w, `

Enter your plex token and plex ip

`) diff --git a/web/settings/settings.html b/web/settings/settings.html index 47df906..fccdc55 100644 --- a/web/settings/settings.html +++ b/web/settings/settings.html @@ -32,6 +32,16 @@

Plex

+

Amazon

+

We can specify a region for the Amazon search on blu-ray.com, the default is UK.

+
+ +
+

MusicBrainz

+

We can specify the url for the server to query, by default we use `https://musicbrainz.org/ws/2`

+
+ +

Spotify

Enter your Spotify client ID and secret to get started. You will need to follow the Spotify documentation to get these. Spotify

+
Back
\ No newline at end of file diff --git a/web/tv/tv.go b/web/tv/tv.go index 280a475..c5c99c3 100644 --- a/web/tv/tv.go +++ b/web/tv/tv.go @@ -66,7 +66,7 @@ func (c TVConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { if lookup == "cinemaParadiso" { tvSearchResults = cinemaparadiso.GetCinemaParadisoTVInParallel(plexTV) } else { - tvSearchResults = amazon.SearchAmazonTVInParallel(plexTV, filters.AudioLanguage) + tvSearchResults = amazon.SearchAmazonTVInParallel(plexTV, filters.AudioLanguage, c.Config.AmazonRegion) } tvJobRunning = false fmt.Printf("\nProcessed %d TV Shows in %v\n", totalTV, time.Since(startTime))