Merge pull request #29 from tphoney/settings_musicbrainz_amazon-region

(feat) allow setting of amazon region and musicbrainz url

TODO

@@ -5,13 +5,13 @@
 - new release for amazon tv series
 - allow amazon tv search for indivdual series
 - new release for cinema-paradiso tv / movie
-- allow setting of amazon region in settings
-- allow setting of musicbrainz url in settings
 
 ## bugs
 
 - music, a-ha/ash doesnt match as an artist why ?
 - when scraping movies, do we stop at the first best match ?
+- speed up plex fetch of movie details
+- speed up plex fetch of tv shows
 
 ## done
 
@@ -40,3 +40,5 @@
 - parallelise amazon search tv/movie
 - move newer show out of amazon and cinema-paradiso, move to web page
 - move language filtering out of plex search, should only happen in web tv & movie web pages
+- allow setting of amazon region in settings
+- allow setting of musicbrainz url in settings

amazon/amazon.go

@@ -26,7 +26,7 @@ var (
 )
 
 // nolint: dupl, nolintlint
-func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string) (searchResults []types.SearchResults) {
+func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language, region string) (searchResults []types.SearchResults) {
 	numberMoviesProcessed = 0
 	ch := make(chan types.SearchResults, len(plexMovies))
 	semaphore := make(chan struct{}, types.ConcurrencyLimit)
@@ -35,7 +35,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string)
 		go func(i int) {
 			semaphore <- struct{}{}
 			defer func() { <-semaphore }()
-			searchAmazonMovie(&plexMovies[i], language, ch)
+			searchAmazonMovie(&plexMovies[i], language, region, ch)
 		}(i)
 	}
 
@@ -51,7 +51,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language string)
 }
 
 // nolint: dupl, nolintlint
-func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language string) (searchResults []types.SearchResults) {
+func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language, region string) (searchResults []types.SearchResults) {
 	numberMoviesProcessed = 0
 	ch := make(chan types.SearchResults, len(plexTVShows))
 	semaphore := make(chan struct{}, types.ConcurrencyLimit)
@@ -60,7 +60,7 @@ func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language string) (
 		go func(i int) {
 			semaphore <- struct{}{}
 			defer func() { <-semaphore }()
-			searchAmazonTV(&plexTVShows[i], language, ch)
+			searchAmazonTV(&plexTVShows[i], language, region, ch)
 		}(i)
 	}
 
@@ -83,15 +83,15 @@ func GetTVJobProgress() int {
 	return numberTVProcessed
 }
 
-func ScrapeTitlesParallel(searchResults []types.SearchResults) (scrapedResults []types.SearchResults) {
+func ScrapeTitlesParallel(searchResults []types.SearchResults, region string) (scrapedResults []types.SearchResults) {
 	numberMoviesProcessed = 0
 	ch := make(chan types.SearchResults, len(searchResults))
 	semaphore := make(chan struct{}, types.ConcurrencyLimit)
 	for i := range searchResults {
 		go func(i int) {
 			semaphore <- struct{}{}
 			defer func() { <-semaphore }()
-			scrapeTitles(&searchResults[i], ch)
+			scrapeTitles(&searchResults[i], region, ch)
 		}(i)
 	}
 
@@ -106,14 +106,14 @@ func ScrapeTitlesParallel(searchResults []types.SearchResults) (scrapedResults [
 	return scrapedResults
 }
 
-func scrapeTitles(searchResult *types.SearchResults, ch chan<- types.SearchResults) {
+func scrapeTitles(searchResult *types.SearchResults, region string, ch chan<- types.SearchResults) {
 	dateAdded := searchResult.PlexMovie.DateAdded
 	for i := range searchResult.MovieSearchResults {
 		// this is to limit the number of requests
 		if !searchResult.MovieSearchResults[i].BestMatch {
 			continue
 		}
-		rawData, err := makeRequest(searchResult.MovieSearchResults[i].URL, "")
+		rawData, err := makeRequest(searchResult.MovieSearchResults[i].URL, region)
 		if err != nil {
 			fmt.Println("scrapeTitle: Error making request:", err)
 			ch <- *searchResult
@@ -134,7 +134,7 @@ func scrapeTitles(searchResult *types.SearchResults, ch chan<- types.SearchResul
 	ch <- *searchResult
 }
 
-func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchResult chan<- types.SearchResults) {
+func searchAmazonMovie(plexMovie *types.PlexMovie, language, region string, movieSearchResult chan<- types.SearchResults) {
 	result := types.SearchResults{}
 	result.PlexMovie = *plexMovie
 	result.SearchURL = ""
@@ -150,7 +150,7 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchR
 	}
 	amazonURL += "&submit=Search&action=search"
 
-	rawData, err := makeRequest(amazonURL, language)
+	rawData, err := makeRequest(amazonURL, region)
 	if err != nil {
 		fmt.Println("searchAmazonMovie: Error making request:", err)
 		movieSearchResult <- result
@@ -163,7 +163,7 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language string, movieSearchR
 	movieSearchResult <- result
 }
 
-func searchAmazonTV(plexTVShow *types.PlexTVShow, language string, tvSearchResult chan<- types.SearchResults) {
+func searchAmazonTV(plexTVShow *types.PlexTVShow, language, region string, tvSearchResult chan<- types.SearchResults) {
 	result := types.SearchResults{}
 	result.PlexTVShow = *plexTVShow
 	result.SearchURL = amazonURL
@@ -179,7 +179,7 @@ func searchAmazonTV(plexTVShow *types.PlexTVShow, language string, tvSearchResul
 	}
 	amazonURL += "&submit=Search&action=search"
 
-	rawData, err := makeRequest(amazonURL, language)
+	rawData, err := makeRequest(amazonURL, region)
 	if err != nil {
 		fmt.Println("searchAmazonTV: Error making request:", err)
 		tvSearchResult <- result
@@ -254,19 +254,13 @@ func findTitlesInResponse(response string, movie bool) (movieResults []types.Mov
 	return movieResults, tvResults
 }
 
-func makeRequest(inputURL, language string) (response string, err error) {
+func makeRequest(inputURL, region string) (response string, err error) {
 	req, err := http.NewRequestWithContext(context.Background(), "GET", inputURL, bytes.NewBuffer([]byte{}))
 
 	req.Header.Set("User-Agent",
 		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
 
-	// this forces results from a specific amazon region
-	switch language {
-	case LanguageGerman:
-		req.Header.Set("Cookie", "country=de;")
-	default:
-		req.Header.Set("Cookie", "country=uk;")
-	}
+	req.Header.Set("Cookie", fmt.Sprintf("country=%s;", region))
 
 	if err != nil {
 		fmt.Println("makeRequest: error creating request:", err)

amazon/amazon_test.go

@@ -8,6 +8,10 @@ import (
 	"github.com/tphoney/plex-lookup/types"
 )
 
+const (
+	amazonRegion = "uk"
+)
+
 var (
 	plexIP    = os.Getenv("PLEX_IP")
 	plexToken = os.Getenv("PLEX_TOKEN")
@@ -39,7 +43,7 @@ func TestFindMoviesInResponse(t *testing.T) {
 }
 
 func TestSearchAmazon(t *testing.T) {
-	result := SearchAmazonMoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "")
+	result := SearchAmazonMoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "", amazonRegion)
 	if len(result) == 0 {
 		t.Errorf("Expected search results, but got none")
 	}
@@ -58,7 +62,7 @@ func TestSearchAmazonTV(t *testing.T) {
 		// Title: "Adventure Time",
 		// Year:  "2010",
 	}
-	result := SearchAmazonTVInParallel([]types.PlexTVShow{show}, "")
+	result := SearchAmazonTVInParallel([]types.PlexTVShow{show}, "", amazonRegion)
 
 	if len(result) == 0 {
 		t.Errorf("Expected search results, but got none")
@@ -81,7 +85,7 @@ func TestScrapeTitlesParallel(t *testing.T) {
 				},
 			},
 		},
-	})
+	}, amazonRegion)
 
 	if len(result) == 0 {
 		t.Errorf("Expected search results, but got none")

cmd/amazon.go

@@ -24,7 +24,7 @@ func performAmazonLookup() {
 	if libraryType == types.PlexMovieType {
 		plexMovies := initializePlexMovies()
 		// lets search movies in amazon
-		searchResults := amazon.SearchAmazonMoviesInParallel(plexMovies, "")
+		searchResults := amazon.SearchAmazonMoviesInParallel(plexMovies, "", amazonRegion)
 		for i := range searchResults {
 			for _, individualResult := range searchResults[i].MovieSearchResults {
 				if individualResult.BestMatch && (individualResult.Format == types.DiskBluray || individualResult.Format == types.Disk4K) {

cmd/root.go

@@ -8,6 +8,10 @@ import (
 	"github.com/tphoney/plex-lookup/types"
 )
 
+const (
+	amazonRegion = "uk"
+)
+
 var (
 	// Used for flags.
 	plexIP             string

cmd/web.go

@@ -25,6 +25,14 @@ func startServer() {
 	config.PlexTVLibraryID = os.Getenv("PLEX_TV_LIBRARY_ID")
 	config.PlexMusicLibraryID = os.Getenv("PLEX_MUSIC_LIBRARY_ID")
 	config.PlexToken = os.Getenv("PLEX_TOKEN")
+	config.AmazonRegion = os.Getenv("AMAZON_REGION")
+	if config.AmazonRegion == "" {
+		config.AmazonRegion = "uk"
+	}
+	config.MusicBrainzURL = os.Getenv("MUSICBRAINZ_URL")
+	if config.MusicBrainzURL == "" {
+		config.MusicBrainzURL = "https://musicbrainz.org/ws/2"
+	}
 	config.SpotifyClientID = os.Getenv("SPOTIFY_CLIENT_ID")
 	config.SpotifyClientSecret = os.Getenv("SPOTIFY_CLIENT_SECRET")
 

musicbrainz/musicbrainz.go

@@ -14,15 +14,13 @@ import (
 // example artist https://musicbrainz.org/artist/83d91898-7763-47d7-b03b-b92132375c47
 
 const (
-	// MusicBrainzURL is the URL for the MusicBrainz API
-	musicBrainzURL = "https://musicbrainz.org/ws/2"
-	agent          = "plex-lookup"
-	agentVersion   = "0.0.1"
-	lookupLimit    = 100
-	lookupTimeout  = 2
+	agent         = "plex-lookup"
+	agentVersion  = "0.0.1"
+	lookupLimit   = 100
+	lookupTimeout = 2
 )
 
-func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.SearchResults, err error) {
+func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist, musicBrainzURL string) (artist types.SearchResults, err error) {
 	artist.PlexMusicArtist = *plexArtist
 	client, err := gomusicbrainz.NewWS2Client(
 		musicBrainzURL, agent, agentVersion, "")
@@ -58,9 +56,9 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se
 	if err != nil {
 		// check for a 503 error
 		if err.Error() == "EOF" {
-			fmt.Println("SearchMusicBrainzArtist rate limit exceeded")
+			fmt.Printf("!")
 			time.Sleep(lookupTimeout * time.Second)
-			return SearchMusicBrainzArtist(plexArtist)
+			return SearchMusicBrainzArtist(plexArtist, musicBrainzURL)
 		}
 	}
 
@@ -75,7 +73,7 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se
 		url := fmt.Sprintf("https://musicbrainz.org/artist/%v", found.ID)
 		found.URL = url
 		// get the albums
-		found.Albums, _ = SearchMusicBrainzAlbums(found.ID)
+		found.Albums, _ = SearchMusicBrainzAlbums(found.ID, musicBrainzURL)
 		artist.MusicSearchResults = append(artist.MusicSearchResults, found)
 		break
 	}
@@ -85,7 +83,7 @@ func SearchMusicBrainzArtist(plexArtist *types.PlexMusicArtist) (artist types.Se
 	return artist, err
 }
 
-func SearchMusicBrainzAlbums(artistID string) (albums []types.MusicAlbumSearchResult, err error) {
+func SearchMusicBrainzAlbums(artistID, musicBrainzURL string) (albums []types.MusicAlbumSearchResult, err error) {
 	client, err := gomusicbrainz.NewWS2Client(
 		musicBrainzURL, agent, agentVersion, "")
 
@@ -98,9 +96,9 @@ func SearchMusicBrainzAlbums(artistID string) (albums []types.MusicAlbumSearchRe
 	resp, err := client.SearchReleaseGroup(queryURL, lookupLimit, -1)
 	if err != nil {
 		if err.Error() == "EOF" {
-			fmt.Println("SearchMusicBrainzAlbums rate limit exceeded")
+			fmt.Printf("!")
 			time.Sleep(lookupTimeout * time.Second)
-			return SearchMusicBrainzAlbums(artistID)
+			return SearchMusicBrainzAlbums(artistID, musicBrainzURL)
 		}
 	}
 	for i := range resp.ReleaseGroups {

musicbrainz/musicbrainz_test.go

@@ -6,6 +6,10 @@ import (
 	"github.com/tphoney/plex-lookup/types"
 )
 
+const (
+	musicBrainzURL = "https://musicbrainz.org/ws/2"
+)
+
 func TestSearchMusicBrainzArtist(t *testing.T) {
 	tests := []struct {
 		name       string
@@ -47,7 +51,7 @@ func TestSearchMusicBrainzArtist(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotArtist, err := SearchMusicBrainzArtist(tt.args)
+			gotArtist, err := SearchMusicBrainzArtist(tt.args, musicBrainzURL)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SearchMusicBrainzArtist() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -66,7 +70,7 @@ func TestSearchMusicBrainzArtist(t *testing.T) {
 // debug test for individual artists
 func TestSearchMusicBrainzArtistDebug(t *testing.T) {
 	artist := &types.PlexMusicArtist{Name: "Aaliyah"}
-	artistSearchResult, err := SearchMusicBrainzArtist(artist)
+	artistSearchResult, err := SearchMusicBrainzArtist(artist, musicBrainzURL)
 	if err != nil {
 		t.Errorf("SearchMusicBrainzArtist() error = %v", err)
 	}
@@ -92,7 +96,7 @@ func TestSearchMusicBrainzAlbums(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotAlbums, err := SearchMusicBrainzAlbums(tt.args.artistID)
+			gotAlbums, err := SearchMusicBrainzAlbums(tt.args.artistID, musicBrainzURL)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SearchMusicBrainzAlbums() error = %v, wantErr %v", err, tt.wantErr)
 				return

types/types.go

@@ -36,6 +36,8 @@ type Configuration struct {
 	PlexMovieLibraryID  string
 	PlexTVLibraryID     string
 	PlexMusicLibraryID  string
+	AmazonRegion        string
+	MusicBrainzURL      string
 	SpotifyClientID     string
 	SpotifyClientSecret string
 }

web/movies/movies.go

@@ -84,10 +84,10 @@ func (c MoviesConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) {
 		if lookup == "cinemaParadiso" {
 			searchResults = cinemaparadiso.GetCinemaParadisoMoviesInParallel(filteredPlexMovies)
 		} else {
-			searchResults = amazon.SearchAmazonMoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage)
+			searchResults = amazon.SearchAmazonMoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage, c.Config.AmazonRegion)
 			// if we are filtering by newer version, we need to search again
 			if lookupFilters.NewerVersion {
-				searchResults = amazon.ScrapeTitlesParallel(searchResults)
+				searchResults = amazon.ScrapeTitlesParallel(searchResults, c.Config.AmazonRegion)
 			}
 		}