diff --git a/TODO b/TODO index 036a4e3..ea971c2 100644 --- a/TODO +++ b/TODO @@ -2,14 +2,13 @@ ## features -- new release for amazon tv series -- allow amazon tv search for indivdual series -- new release for cinema-paradiso tv +- allow the use of playlists for finding music / TV / movies ## bugs -- music, a-ha/ash doesnt match as an artist why ? - improve cinema-paradiso movie scrape, many search results are the same page. wasted processing +- flatten amazon tv search results +- improve best match for tv series ## done @@ -43,3 +42,6 @@ - speed up plex fetch of movie details - speed up plex fetch of tv shows - when scraping movies, do we stop at the first best match ? +- new release for cinema-paradiso tv +- new release for amazon tv series +- allow amazon tv search for indivdual series diff --git a/amazon/amazon.go b/amazon/amazon.go index d98abeb..497ece5 100644 --- a/amazon/amazon.go +++ b/amazon/amazon.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "regexp" + "strconv" "strings" "time" @@ -23,10 +24,56 @@ const ( var ( numberMoviesProcessed int = 0 numberTVProcessed int = 0 + //nolint: mnd + seasonNumberToInt = map[string]int{ + "one": 1, + "two": 2, + "three": 3, + "four": 4, + "five": 5, + "six": 6, + "seven": 7, + "eight": 8, + "nine": 9, + "ten": 10, + "eleven": 11, + "twelve": 12, + "thirteen": 13, + "fourteen": 14, + "fifteen": 15, + "sixteen": 16, + "seventeen": 17, + "eighteen": 18, + "nineteen": 19, + "twenty": 20, + } + //nolint: mnd + ordinalNumberToSeason = map[string]int{ + "first season": 1, + "second season": 2, + "third season": 3, + "fourth season": 4, + "fifth season": 5, + "sixth season": 6, + "seventh season": 7, + "eighth season": 8, + "ninth season": 9, + "tenth season": 10, + "eleventh season": 11, + "twelfth season": 12, + "thirteenth season": 13, + "fourteenth season": 14, + "fifteenth season": 15, + "sixteenth season": 16, + "seventeenth season": 17, + "eighteenth season": 18, + "nineteenth season": 19, + "twentieth season": 20, + } ) // nolint: dupl, nolintlint -func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language, region string) (searchResults []types.SearchResults) { +func MoviesInParallel(plexMovies []types.PlexMovie, language, region string) (searchResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(plexMovies)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -35,7 +82,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language, region go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - searchAmazonMovie(&plexMovies[i], language, region, ch) + searchMovie(&plexMovies[i], language, region, ch) }(i) } @@ -51,7 +98,7 @@ func SearchAmazonMoviesInParallel(plexMovies []types.PlexMovie, language, region } // nolint: dupl, nolintlint -func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language, region string) (searchResults []types.SearchResults) { +func TVInParallel(plexTVShows []types.PlexTVShow, language, region string) (searchResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(plexTVShows)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -60,7 +107,7 @@ func SearchAmazonTVInParallel(plexTVShows []types.PlexTVShow, language, region s go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - searchAmazonTV(&plexTVShows[i], language, region, ch) + searchTV(&plexTVShows[i], language, region, ch) }(i) } @@ -83,15 +130,24 @@ func GetTVJobProgress() int { return numberTVProcessed } -func ScrapeTitlesParallel(searchResults []types.SearchResults, region string) (scrapedResults []types.SearchResults) { - numberMoviesProcessed = 0 +func ScrapeTitlesParallel(searchResults []types.SearchResults, region string, isTV bool) (scrapedResults []types.SearchResults) { + // are we tv or movie + if isTV { + numberTVProcessed = 0 + } else { + numberMoviesProcessed = 0 + } ch := make(chan types.SearchResults, len(searchResults)) semaphore := make(chan struct{}, types.ConcurrencyLimit) for i := range searchResults { go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - scrapeTitles(&searchResults[i], region, ch) + if isTV { + scrapeTVTitles(&searchResults[i], region, ch) + } else { + scrapeMovieTitles(&searchResults[i], region, ch) + } }(i) } @@ -99,14 +155,23 @@ func ScrapeTitlesParallel(searchResults []types.SearchResults, region string) (s for range searchResults { result := <-ch scrapedResults = append(scrapedResults, result) - numberMoviesProcessed++ + if isTV { + numberTVProcessed++ + } else { + numberMoviesProcessed++ + } } - numberMoviesProcessed = 0 - fmt.Println("amazon Movie titles scraped:", len(scrapedResults)) + if isTV { + numberTVProcessed = 0 + } else { + numberMoviesProcessed = 0 + } + fmt.Println("amazon titles scraped:", len(scrapedResults)) return scrapedResults } -func scrapeTitles(searchResult *types.SearchResults, region string, ch chan<- types.SearchResults) { +// nolint: dupl, nolintlint +func scrapeMovieTitles(searchResult *types.SearchResults, region string, ch chan<- types.SearchResults) { dateAdded := searchResult.PlexMovie.DateAdded for i := range searchResult.MovieSearchResults { // this is to limit the number of requests @@ -134,7 +199,36 @@ func scrapeTitles(searchResult *types.SearchResults, region string, ch chan<- ty ch <- *searchResult } -func searchAmazonMovie(plexMovie *types.PlexMovie, language, region string, movieSearchResult chan<- types.SearchResults) { +// nolint: dupl, nolintlint +func scrapeTVTitles(searchResult *types.SearchResults, region string, ch chan<- types.SearchResults) { + dateAdded := searchResult.PlexTVShow.DateAdded + for i := range searchResult.TVSearchResults { + // this is to limit the number of requests + if !searchResult.TVSearchResults[i].BestMatch { + continue + } + rawData, err := makeRequest(searchResult.TVSearchResults[i].URL, region) + if err != nil { + fmt.Println("scrapeTitle: Error making request:", err) + ch <- *searchResult + return + } + // Find the release date + searchResult.TVSearchResults[i].ReleaseDate = time.Time{} // default to zero time + r := regexp.MustCompile(`(.*?)`) + match := r.FindStringSubmatch(rawData) + if match != nil { + stringDate := match[1] + searchResult.TVSearchResults[i].ReleaseDate, _ = time.Parse("Jan 02, 2006", stringDate) + } + if searchResult.TVSearchResults[i].ReleaseDate.After(dateAdded) { + searchResult.TVSearchResults[i].NewRelease = true + } + } + ch <- *searchResult +} + +func searchMovie(plexMovie *types.PlexMovie, language, region string, movieSearchResult chan<- types.SearchResults) { result := types.SearchResults{} result.PlexMovie = *plexMovie @@ -152,7 +246,7 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language, region string, movi result.SearchURL = amazonURL rawData, err := makeRequest(amazonURL, region) if err != nil { - fmt.Println("searchAmazonMovie: Error making request:", err) + fmt.Println("searchMovie: Error making request:", err) movieSearchResult <- result return } @@ -163,12 +257,12 @@ func searchAmazonMovie(plexMovie *types.PlexMovie, language, region string, movi movieSearchResult <- result } -func searchAmazonTV(plexTVShow *types.PlexTVShow, language, region string, tvSearchResult chan<- types.SearchResults) { +func searchTV(plexTVShow *types.PlexTVShow, language, region string, tvSearchResult chan<- types.SearchResults) { result := types.SearchResults{} result.PlexTVShow = *plexTVShow result.SearchURL = amazonURL - urlEncodedTitle := url.QueryEscape(fmt.Sprintf("%s complete series", plexTVShow.Title)) // complete series + urlEncodedTitle := url.QueryEscape(plexTVShow.Title) amazonURL := amazonURL + urlEncodedTitle // this searches for the movie in a language switch language { @@ -181,7 +275,7 @@ func searchAmazonTV(plexTVShow *types.PlexTVShow, language, region string, tvSea rawData, err := makeRequest(amazonURL, region) if err != nil { - fmt.Println("searchAmazonTV: Error making request:", err) + fmt.Println("searchTV: Error making request:", err) tvSearchResult <- result return } @@ -202,7 +296,6 @@ func findTitlesInResponse(response string, movie bool) (movieResults []types.Mov } response = response[startIndex:] endIndex := strings.Index(response, ``) - // If both start and end index are found if endIndex != -1 { // Extract the entry @@ -235,13 +328,14 @@ func findTitlesInResponse(response string, movie bool) (movieResults []types.Mov movieResults = append(movieResults, types.MovieSearchResult{ URL: returnURL, Format: format, Year: year, FoundTitle: foundTitle, UITitle: format}) } else { - boxSet := false - if strings.Contains(foundTitle, ": The Complete Series") { - foundTitle = strings.TrimSuffix(foundTitle, ": The Complete Series") - boxSet = true - } - tvResults = append(tvResults, types.TVSearchResult{ - URL: returnURL, Format: []string{format}, Year: year, FoundTitle: foundTitle, UITitle: foundTitle, BoxSet: boxSet}) + decipheredTitle, number, boxSet := decipherTVName(foundTitle) + // split year + splitYear := strings.Split(year, "-") + year = splitYear[0] + tvResult := types.TVSearchResult{ + URL: returnURL, Format: []string{format}, Year: year, FoundTitle: decipheredTitle, UITitle: decipheredTitle} + tvResult.Seasons = append(tvResult.Seasons, types.TVSeasonResult{URL: returnURL, Format: format, Number: number, BoxSet: boxSet}) + tvResults = append(tvResults, tvResult) } } // remove the movie entry from the response @@ -291,3 +385,46 @@ func makeRequest(inputURL, region string) (response string, err error) { rawResponse := string(body) return rawResponse, nil } + +func decipherTVName(name string) (title string, number int, boxset bool) { + parts := strings.Split(name, ":") + title = parts[0] + if len(parts) == 1 { + //nolint: gocritic + // fmt.Printf("warn: decipherTVName, no colon%q\n", name) + return title, -1, false + } + // everything after the first colon + seasonBlock := strings.Join(parts[1:], "") + seasonBlock = strings.ToLower(seasonBlock) + if strings.Contains(seasonBlock, "complete series") || strings.Contains(seasonBlock, "complete seasons") { + return title, number, true + } + // does the second part have a number as an integer or as a word. + r := regexp.MustCompile(`seasons?\ (\d+)`) + match := r.FindStringSubmatch(seasonBlock) + if len(match) > 1 { + // var err error + number, _ = strconv.Atoi(match[1]) + //nolint: gocritic + // if err != nil { + // fmt.Printf("warn: decipherTVName, integer not converted%q\n", name) + // } + return title, number, false + } + + for k, v := range seasonNumberToInt { + if strings.Contains(seasonBlock, ("season "+k)) || strings.Contains(seasonBlock, ("seasons "+k)) { + return title, v, false + } + } + + for k, v := range ordinalNumberToSeason { + if strings.Contains(seasonBlock, k) { + return title, v, false + } + } + //nolint: gocritic + // fmt.Printf("warn: decipherTVName, got to the end%q\n", name) + return title, -1, false +} diff --git a/amazon/amazon_test.go b/amazon/amazon_test.go index e40a9c2..97a1d6f 100644 --- a/amazon/amazon_test.go +++ b/amazon/amazon_test.go @@ -43,7 +43,7 @@ func TestFindMoviesInResponse(t *testing.T) { } func TestSearchAmazon(t *testing.T) { - result := SearchAmazonMoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "", amazonRegion) + result := MoviesInParallel([]types.PlexMovie{{Title: "napoleon dynamite", Year: "2004"}}, "", amazonRegion) if len(result) == 0 { t.Errorf("Expected search results, but got none") } @@ -57,19 +57,20 @@ func TestSearchAmazonTV(t *testing.T) { t.Skip("ACCEPTANCE TEST: PLEX environment variables not set") } show := types.PlexTVShow{ - Title: "Friends", - Year: "1994", + // Title: "Friends", + // Year: "1994", // Title: "Charmed", // Year: "1998", // Title: "Adventure Time", // Year: "2010", + Title: "Star Trek: Enterprise", + Year: "2001", } - result := SearchAmazonTVInParallel([]types.PlexTVShow{show}, "", amazonRegion) + result := TVInParallel([]types.PlexTVShow{show}, "", amazonRegion) if len(result) == 0 { t.Errorf("Expected search results, but got none") } - fmt.Println(result) } func TestScrapeTitlesParallel(t *testing.T) { @@ -87,7 +88,7 @@ func TestScrapeTitlesParallel(t *testing.T) { }, }, }, - }, amazonRegion) + }, amazonRegion, false) if len(result) == 0 { t.Errorf("Expected search results, but got none") diff --git a/cinemaparadiso/cinemaparadiso.go b/cinemaparadiso/cinemaparadiso.go index 4a2b29e..eda28aa 100644 --- a/cinemaparadiso/cinemaparadiso.go +++ b/cinemaparadiso/cinemaparadiso.go @@ -27,7 +27,7 @@ var ( ) // nolint: dupl, nolintlint -func GetCinemaParadisoMoviesInParallel(plexMovies []types.PlexMovie) (searchResults []types.SearchResults) { +func MoviesInParallel(plexMovies []types.PlexMovie) (searchResults []types.SearchResults) { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(plexMovies)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -50,7 +50,7 @@ func GetCinemaParadisoMoviesInParallel(plexMovies []types.PlexMovie) (searchResu return searchResults } -func ScrapeMovieTitlesParallel(searchResults []types.SearchResults) []types.SearchResults { +func ScrapeMoviesParallel(searchResults []types.SearchResults) []types.SearchResults { numberMoviesProcessed = 0 ch := make(chan types.SearchResults, len(searchResults)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -73,7 +73,7 @@ func ScrapeMovieTitlesParallel(searchResults []types.SearchResults) []types.Sear } // nolint: dupl, nolintlint -func GetCinemaParadisoTVInParallel(plexTVShows []types.PlexTVShow) (searchResults []types.SearchResults) { +func TVInParallel(plexTVShows []types.PlexTVShow) (searchResults []types.SearchResults) { ch := make(chan types.SearchResults, len(plexTVShows)) semaphore := make(chan struct{}, types.ConcurrencyLimit) @@ -81,7 +81,7 @@ func GetCinemaParadisoTVInParallel(plexTVShows []types.PlexTVShow) (searchResult go func(i int) { semaphore <- struct{}{} defer func() { <-semaphore }() - searchCinemaParadisoTV(&plexTVShows[i], ch) + searchTVShow(&plexTVShows[i], ch) }(i) } @@ -110,7 +110,7 @@ func searchCinemaParadisoMovie(plexMovie *types.PlexMovie, movieSearchResult cha result.SearchURL = cinemaparadisoSearchURL + "?form-search-field=" + urlEncodedTitle rawData, err := makeRequest(result.SearchURL, http.MethodPost, fmt.Sprintf("form-search-field=%s", urlEncodedTitle)) if err != nil { - fmt.Println("Error making web request:", err) + fmt.Println("searchCinemaParadisoMovie:", err) movieSearchResult <- result return } @@ -122,14 +122,14 @@ func searchCinemaParadisoMovie(plexMovie *types.PlexMovie, movieSearchResult cha } func scrapeMovieTitle(result *types.SearchResults, movieSearchResult chan<- types.SearchResults) { - // now we can get the series information for each best match + // now we can get the season information for each best match for i := range result.MovieSearchResults { if !result.MovieSearchResults[i].BestMatch { continue } rawData, err := makeRequest(result.MovieSearchResults[i].URL, http.MethodGet, "") if err != nil { - fmt.Println("Error making web request:", err) + fmt.Println("scrapeMovieTitle:", err) movieSearchResult <- *result return } @@ -162,14 +162,14 @@ func scrapeMovieTitle(result *types.SearchResults, movieSearchResult chan<- type movieSearchResult <- *result } -func searchCinemaParadisoTV(plexTVShow *types.PlexTVShow, tvSearchResult chan<- types.SearchResults) { +func searchTVShow(plexTVShow *types.PlexTVShow, tvSearchResult chan<- types.SearchResults) { result := types.SearchResults{} urlEncodedTitle := url.QueryEscape(plexTVShow.Title) result.PlexTVShow = *plexTVShow result.SearchURL = cinemaparadisoSearchURL + "?form-search-field=" + urlEncodedTitle rawData, err := makeRequest(result.SearchURL, http.MethodPost, fmt.Sprintf("form-search-field=%s", urlEncodedTitle)) if err != nil { - fmt.Println("searchCinemaParadisoTV: Error making web request:", err) + fmt.Println("searchTVShow: Error making web request:", err) tvSearchResult <- result return } @@ -177,65 +177,70 @@ func searchCinemaParadisoTV(plexTVShow *types.PlexTVShow, tvSearchResult chan<- _, tvFound := findTitlesInResponse(rawData, false) result.TVSearchResults = tvFound result = utils.MarkBestMatch(&result) - // now we can get the series information for each best match + // now we can get the season information for each best match for i := range result.TVSearchResults { if result.TVSearchResults[i].BestMatch { - result.TVSearchResults[i].Seasons, _ = findTVSeriesInfo(result.TVSearchResults[i].URL) + result.TVSearchResults[i].Seasons, _ = findTVSeasonInfo(result.TVSearchResults[i].URL) } } tvSearchResult <- result } -func findTVSeriesInfo(seriesURL string) (tvSeries []types.TVSeasonResult, err error) { +func findTVSeasonInfo(seriesURL string) (tvSeasons []types.TVSeasonResult, err error) { // make a request to the url rawData, err := makeRequest(seriesURL, http.MethodGet, "") if err != nil { - fmt.Println("findTVSeriesInfo: Error making web request:", err) - return tvSeries, err + fmt.Println("findTVSeasonInfo: Error making web request:", err) + return tvSeasons, err } - tvSeries = findTVSeriesInResponse(rawData) - return tvSeries, nil + tvSeasons = findTVSeasonsInResponse(rawData) + return tvSeasons, nil } -func findTVSeriesInResponse(response string) (tvSeries []types.TVSeasonResult) { +func findTVSeasonsInResponse(response string) (tvSeasons []types.TVSeasonResult) { // look for the series in the response r := regexp.MustCompile(`
  • `) match := r.FindAllStringSubmatch(response, -1) for i, m := range match { - tvSeries = append(tvSeries, types.TVSeasonResult{Number: i, URL: m[1]}) + tvSeasons = append(tvSeasons, types.TVSeasonResult{Number: i, URL: m[1]}) } // remove the first entry as it is general information - results := make([]types.TVSeasonResult, 0, len(tvSeries)) - if len(tvSeries) > 0 { - tvSeries = tvSeries[1:] + scrapedTVSeasonResults := make([]types.TVSeasonResult, 0, len(tvSeasons)) + if len(tvSeasons) > 0 { + tvSeasons = tvSeasons[1:] - for i := range tvSeries { - seriesResult, err := makeSeriesRequest(tvSeries[i]) + for i := range tvSeasons { + detailedSeasonResults, err := makeSeasonRequest(tvSeasons[i]) if err != nil { - fmt.Println("Error making series request:", err) + fmt.Println("findTVSeasonsInResponse: Error making season request:", err) continue } - results = append(results, seriesResult) + scrapedTVSeasonResults = append(scrapedTVSeasonResults, detailedSeasonResults...) } } // sort the results by number - sort.Slice(results, func(i, j int) bool { - return results[i].Number < results[j].Number + sort.Slice(scrapedTVSeasonResults, func(i, j int) bool { + return scrapedTVSeasonResults[i].Number < scrapedTVSeasonResults[j].Number }) - return results + return scrapedTVSeasonResults } -func makeSeriesRequest(tv types.TVSeasonResult) (types.TVSeasonResult, error) { +func makeSeasonRequest(tv types.TVSeasonResult) (result []types.TVSeasonResult, err error) { rawData, err := makeRequest(cinemaparadisoSeriesURL, http.MethodPost, fmt.Sprintf("FilmID=%s", tv.URL)) if err != nil { - return tv, fmt.Errorf("makeSeriesRequest: error making request: %w", err) + return result, fmt.Errorf("makeSeasonRequest: error making request: %w", err) } + // os.WriteFile("series.html", []byte(rawData), 0644) // write the raw data to a file r := regexp.MustCompile(`{.."Media..":.."(.*?)",.."ReleaseDate..":.."(.*?)"}`) // Find all matches matches := r.FindAllStringSubmatch(rawData, -1) + // there will be multiple formats for each season eg https://www.cinemaparadiso.co.uk/rentals/airwolf-171955.html#dvd for _, match := range matches { - tv.Format = append(tv.Format, strings.ReplaceAll(match[1], "\\", "")) + newSeason := types.TVSeasonResult{} + newSeason.Number = tv.Number + newSeason.Format = strings.ReplaceAll(match[1], "\\", "") + newSeason.URL = fmt.Sprintf("https://www.cinemaparadiso.co.uk/rentals/%s.html#%s", tv.URL, newSeason.Format) // strip slashes from the date date := strings.ReplaceAll(match[2], "\\", "") var releaseDate time.Time @@ -243,9 +248,10 @@ func makeSeriesRequest(tv types.TVSeasonResult) (types.TVSeasonResult, error) { if err != nil { releaseDate = time.Time{} } - tv.ReleaseDate = releaseDate + newSeason.ReleaseDate = releaseDate + result = append(result, newSeason) } - return tv, nil + return result, nil } func findTitlesInResponse(response string, movie bool) (movieResults []types.MovieSearchResult, tvResults []types.TVSearchResult) { diff --git a/cinemaparadiso/cinemaparadiso_test.go b/cinemaparadiso/cinemaparadiso_test.go index d6cff49..ee9012d 100644 --- a/cinemaparadiso/cinemaparadiso_test.go +++ b/cinemaparadiso/cinemaparadiso_test.go @@ -67,27 +67,37 @@ func TestFindTVSeriesInResponse(t *testing.T) { t.Errorf("Error reading testdata/friends.html: %s", err) } - tvSeries := findTVSeriesInResponse(string(rawdata)) + tvSeries := findTVSeasonsInResponse(string(rawdata)) - if len(tvSeries) != 10 { - t.Errorf("Expected 10 tv series, but got %d", len(tvSeries)) + if len(tvSeries) != 20 { + t.Errorf("Expected 20 tv series, but got %d", len(tvSeries)) } // check the first tv series - if tvSeries[0].Number != 1 { + if tvSeries[1].Number != 1 { t.Errorf("Expected number 1, but got %d", tvSeries[0].Number) } - expected := time.Date(2012, time.November, 12, 0, 0, 0, 0, time.UTC) + expected := time.Date(2004, time.October, 25, 0, 0, 0, 0, time.UTC) if tvSeries[0].ReleaseDate.Compare(expected) != 0 { t.Errorf("Expected date %s, but got %s", expected, tvSeries[0].ReleaseDate) } if tvSeries[0].Number != 1 { t.Errorf("Expected number 1, but got %d", tvSeries[0].Number) } - if tvSeries[0].Format[0] != types.DiskDVD { - t.Errorf("Expected format %s, but got %s", types.DiskDVD, tvSeries[0].Format[0]) + if tvSeries[0].Format != types.DiskDVD { + t.Errorf("Expected dvd, but got %s", tvSeries[0].Format) + } + if tvSeries[1].Number != 1 { + t.Errorf("Expected number 1, but got %d", tvSeries[0].Number) + } + expected = time.Date(2012, time.November, 12, 0, 0, 0, 0, time.UTC) + if tvSeries[1].ReleaseDate.Compare(expected) != 0 { + t.Errorf("Expected date %s, but got %s", expected, tvSeries[0].ReleaseDate) + } + if tvSeries[1].Number != 1 { + t.Errorf("Expected number 1, but got %d", tvSeries[0].Number) } - if tvSeries[0].Format[1] != types.DiskBluray { - t.Errorf("Expected format %s, but got %s", types.DiskBluray, tvSeries[0].Format[1]) + if tvSeries[1].Format != types.DiskBluray { + t.Errorf("Expected Blu-ray, but got %s", tvSeries[0].Format) } } @@ -104,7 +114,7 @@ func TestSearchCinemaParadisoTV(t *testing.T) { Year: "2010", } ch := make(chan types.SearchResults, 1) - searchCinemaParadisoTV(&show, ch) + searchTVShow(&show, ch) got := <-ch if len(got.TVSearchResults) == 0 { @@ -154,7 +164,7 @@ func TestScrapeMovieTitlesParallel(t *testing.T) { }, } - detailedSearchResults := ScrapeMovieTitlesParallel(searchResults) + detailedSearchResults := ScrapeMoviesParallel(searchResults) if len(detailedSearchResults) != len(searchResults) { t.Errorf("Expected %d detailed search results, but got %d", len(searchResults), len(detailedSearchResults)) diff --git a/cmd/amazon.go b/cmd/amazon.go index 59e8e1b..6aaf139 100644 --- a/cmd/amazon.go +++ b/cmd/amazon.go @@ -24,7 +24,7 @@ func performAmazonLookup() { if libraryType == types.PlexMovieType { plexMovies := initializePlexMovies() // lets search movies in amazon - searchResults := amazon.SearchAmazonMoviesInParallel(plexMovies, "", amazonRegion) + searchResults := amazon.MoviesInParallel(plexMovies, "", amazonRegion) for i := range searchResults { for _, individualResult := range searchResults[i].MovieSearchResults { if individualResult.BestMatch && (individualResult.Format == types.DiskBluray || individualResult.Format == types.Disk4K) { diff --git a/cmd/cinemaparadiso.go b/cmd/cinemaparadiso.go index 29a3869..96c1c5e 100644 --- a/cmd/cinemaparadiso.go +++ b/cmd/cinemaparadiso.go @@ -24,7 +24,7 @@ func performCinemaParadisoLookup() { if libraryType == types.PlexMovieType { plexMovies := initializePlexMovies() // lets search movies in cinemaparadiso - searchResults := cinemaparadiso.GetCinemaParadisoMoviesInParallel(plexMovies) + searchResults := cinemaparadiso.MoviesInParallel(plexMovies) // if hit, and contains any format that isnt dvd, print the movie for i := range searchResults { for _, individualResult := range searchResults[i].MovieSearchResults { diff --git a/plex/plex.go b/plex/plex.go index 21087d1..3328cda 100644 --- a/plex/plex.go +++ b/plex/plex.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "slices" + "sort" "strconv" "strings" "time" @@ -764,6 +765,9 @@ func GetPlexTV(ipAddress, libraryID, plexToken string) (tvShowList []types.PlexT var filteredTVShows []types.PlexTVShow for i := range tvShowList { if len(tvShowList[i].Seasons) > 0 { + // set the first and last episode air dates + tvShowList[i].FirstEpisodeAired = tvShowList[i].Seasons[0].FirstEpisodeAired + tvShowList[i].LastEpisodeAired = tvShowList[i].Seasons[len(tvShowList[i].Seasons)-1].LastEpisodeAired filteredTVShows = append(filteredTVShows, tvShowList[i]) } } @@ -810,10 +814,16 @@ func getPlexTVSeasons(ipAddress, plexToken, ratingKey string) (seasonList []type } // now we have all of the resolutions for the episodes detailedSeasons[i].LowestResolution = findLowestResolution(listOfResolutions) - // get the last episode added date + // get the dates for the season detailedSeasons[i].LastEpisodeAdded = detailedSeasons[i].Episodes[len(detailedSeasons[i].Episodes)-1].DateAdded + detailedSeasons[i].LastEpisodeAired = detailedSeasons[i].Episodes[len(detailedSeasons[i].Episodes)-1].OriginallyAired + detailedSeasons[i].FirstEpisodeAired = detailedSeasons[i].Episodes[0].OriginallyAired filteredSeasons = append(filteredSeasons, detailedSeasons[i]) } + // sort the seasons by season number + sort.Slice(filteredSeasons, func(i, j int) bool { + return filteredSeasons[i].Number < filteredSeasons[j].Number + }) return filteredSeasons } @@ -877,16 +887,15 @@ func extractTVEpisodes(xmlString string) (episodeList []types.PlexTVEpisode) { } for i := range container.Video { - intTime, err := strconv.ParseInt(container.Video[i].AddedAt, 10, 64) - var parsedDate time.Time + dateAdded := parsePlexDate(container.Video[i].AddedAt) + // "2017-04-21" + originallyAired, err := time.Parse("2006-01-02", container.Video[i].OriginallyAvailableAt) if err != nil { - parsedDate = time.Time{} - } else { - parsedDate = time.Unix(intTime, 0) + originallyAired = time.Time{} } episodeList = append(episodeList, types.PlexTVEpisode{ Title: container.Video[i].Title, Resolution: container.Video[i].Media.VideoResolution, - Index: container.Video[i].Index, DateAdded: parsedDate}) + Index: container.Video[i].Index, DateAdded: dateAdded, OriginallyAired: originallyAired}) } return episodeList } diff --git a/types/types.go b/types/types.go index e7dcab6..b2472ef 100644 --- a/types/types.go +++ b/types/types.go @@ -75,27 +75,32 @@ type MovieSearchResult struct { // ============================================================================================================== type PlexTVShow struct { - Title string - Year string - RatingKey string - DateAdded time.Time - Seasons []PlexTVSeason + Title string + Year string + RatingKey string + DateAdded time.Time + FirstEpisodeAired time.Time + LastEpisodeAired time.Time + Seasons []PlexTVSeason } type PlexTVSeason struct { - Title string - Number int - RatingKey string - LowestResolution string - LastEpisodeAdded time.Time - Episodes []PlexTVEpisode + Title string + Number int + RatingKey string + LowestResolution string + LastEpisodeAdded time.Time + FirstEpisodeAired time.Time + LastEpisodeAired time.Time + Episodes []PlexTVEpisode } type PlexTVEpisode struct { - Title string - Index string - Resolution string - DateAdded time.Time + Title string + Index string + Resolution string + DateAdded time.Time + OriginallyAired time.Time } type TVSearchResult struct { @@ -107,14 +112,14 @@ type TVSearchResult struct { Year string ReleaseDate time.Time NewRelease bool - BoxSet bool Seasons []TVSeasonResult } type TVSeasonResult struct { Number int URL string - Format []string + Format string + BoxSet bool ReleaseDate time.Time } diff --git a/utils/utils.go b/utils/utils.go index 7256cf0..f5575a8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -25,6 +25,7 @@ func MarkBestMatch(search *types.SearchResults) types.SearchResults { } } } + expectedYear = YearToDate(search.PlexTVShow.Year) for i := range search.TVSearchResults { resultYear := YearToDate(search.TVSearchResults[i].Year) if search.TVSearchResults[i].FoundTitle == search.PlexTVShow.Title && WitinOneYear(resultYear.Year(), expectedYear.Year()) { diff --git a/web/movies/movies.go b/web/movies/movies.go index f4bd8b6..cea43ae 100644 --- a/web/movies/movies.go +++ b/web/movies/movies.go @@ -82,15 +82,15 @@ func (c MoviesConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { go func() { startTime := time.Now() if lookup == "cinemaParadiso" { - searchResults = cinemaparadiso.GetCinemaParadisoMoviesInParallel(filteredPlexMovies) + searchResults = cinemaparadiso.MoviesInParallel(filteredPlexMovies) if lookupFilters.NewerVersion { - searchResults = cinemaparadiso.ScrapeMovieTitlesParallel(searchResults) + searchResults = cinemaparadiso.ScrapeMoviesParallel(searchResults) } } else { - searchResults = amazon.SearchAmazonMoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage, c.Config.AmazonRegion) + searchResults = amazon.MoviesInParallel(filteredPlexMovies, lookupFilters.AudioLanguage, c.Config.AmazonRegion) // if we are filtering by newer version, we need to search again if lookupFilters.NewerVersion { - searchResults = amazon.ScrapeTitlesParallel(searchResults, c.Config.AmazonRegion) + searchResults = amazon.ScrapeTitlesParallel(searchResults, c.Config.AmazonRegion, false) } } diff --git a/web/tv/tv.go b/web/tv/tv.go index 5325460..b2ea301 100644 --- a/web/tv/tv.go +++ b/web/tv/tv.go @@ -51,7 +51,7 @@ func (c TVConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { } filters = newFilters //nolint: gocritic - // plexTV = plexTV[:30] + //plexTV = plexTV[:10] //lint: gocritic tvJobRunning = true @@ -64,9 +64,10 @@ func (c TVConfig) ProcessHTML(w http.ResponseWriter, r *http.Request) { go func() { startTime := time.Now() if lookup == "cinemaParadiso" { - tvSearchResults = cinemaparadiso.GetCinemaParadisoTVInParallel(plexTV) + tvSearchResults = cinemaparadiso.TVInParallel(plexTV) } else { - tvSearchResults = amazon.SearchAmazonTVInParallel(plexTV, filters.AudioLanguage, c.Config.AmazonRegion) + tvSearchResults = amazon.TVInParallel(plexTV, filters.AudioLanguage, c.Config.AmazonRegion) + tvSearchResults = amazon.ScrapeTitlesParallel(tvSearchResults, c.Config.AmazonRegion, true) } tvJobRunning = false fmt.Printf("\nProcessed %d TV Shows in %v\n", totalTV, time.Since(startTime)) @@ -99,30 +100,25 @@ func renderTVTable(searchResults []types.SearchResults) (tableRows string) { tableRows = `Plex TitleBlu-ray Seasons4K-ray SeasonsDisc` //nolint: lll for i := range searchResults { // build up plex season / resolution row - seasony := "Season:" - for _, season := range searchResults[i].PlexTVShow.Seasons { - seasony += fmt.Sprintf(" %d@%s,", season.Number, season.LowestResolution) + plexSeasonsString := "Season:" + for j := range searchResults[i].PlexTVShow.Seasons { + plexSeasonsString += fmt.Sprintf(" %d@%s,", + searchResults[i].PlexTVShow.Seasons[j].Number, searchResults[i].PlexTVShow.Seasons[j].LowestResolution) } - seasony = seasony[:len(seasony)-1] // remove trailing comma + plexSeasonsString = plexSeasonsString[:len(plexSeasonsString)-1] // remove trailing comma tableRows += fmt.Sprintf( `%s [%v]
    %s    %d%d`, - searchResults[i].SearchURL, searchResults[i].PlexTVShow.Title, searchResults[i].PlexTVShow.Year, seasony, + searchResults[i].SearchURL, searchResults[i].PlexTVShow.Title, searchResults[i].PlexTVShow.Year, plexSeasonsString, searchResults[i].MatchesBluray, searchResults[i].Matches4k) if (searchResults[i].MatchesBluray + searchResults[i].Matches4k) > 0 { tableRows += "" for j := range searchResults[i].TVSearchResults { if searchResults[i].TVSearchResults[j].BestMatch { - if searchResults[i].TVSearchResults[j].BoxSet { - tableRows += fmt.Sprintf(`%s Box Set
    `, - searchResults[i].TVSearchResults[j].URL, searchResults[i].TVSearchResults[j].Format[0]) - } else { - for _, season := range searchResults[i].TVSearchResults[j].Seasons { - disks := fmt.Sprintf("%v", season.Format) - tableRows += fmt.Sprintf( - `Season %d: %v`, - searchResults[i].TVSearchResults[j].URL, season.Number, disks) - tableRows += "
    " - } + for _, season := range searchResults[i].TVSearchResults[j].Seasons { + tableRows += fmt.Sprintf( + `Season %d: %v`, + searchResults[i].TVSearchResults[j].URL, season.Number, season.Format) + tableRows += "
    " } } } @@ -148,10 +144,11 @@ func removeOwnedTVSeasons(searchResults []types.SearchResults) []types.SearchRes if len(searchResults[i].TVSearchResults) > 0 { tvSeasonsToRemove := make([]types.TVSeasonResult, 0) // iterate over plex tv season - for _, plexSeasons := range searchResults[i].PlexTVShow.Seasons { + for plexSeasonPointer := range searchResults[i].PlexTVShow.Seasons { // iterate over search results for _, searchSeasons := range searchResults[i].TVSearchResults[0].Seasons { - if searchSeasons.Number == plexSeasons.Number && !discBeatsPlexResolution(plexSeasons.LowestResolution, searchSeasons.Format) { + if searchSeasons.Number == searchResults[i].PlexTVShow.Seasons[plexSeasonPointer].Number && + !discBeatsPlexResolution(searchResults[i].PlexTVShow.Seasons[plexSeasonPointer].LowestResolution, searchSeasons.Format) { tvSeasonsToRemove = append(tvSeasonsToRemove, searchSeasons) } } @@ -167,10 +164,10 @@ func removeOldDiscReleases(searchResults []types.SearchResults) []types.SearchRe if len(searchResults[i].TVSearchResults) > 0 { tvSeasonsToRemove := make([]types.TVSeasonResult, 0) // iterate over plex tv season - for _, plexSeasons := range searchResults[i].PlexTVShow.Seasons { + for plesSeasonPointer := range searchResults[i].PlexTVShow.Seasons { // iterate over search results for _, searchSeasons := range searchResults[i].TVSearchResults[0].Seasons { - if searchSeasons.ReleaseDate.Compare(plexSeasons.LastEpisodeAdded) == 1 { + if searchSeasons.ReleaseDate.Compare(searchResults[i].PlexTVShow.Seasons[plesSeasonPointer].LastEpisodeAdded) == 1 { tvSeasonsToRemove = append(tvSeasonsToRemove, searchSeasons) } } @@ -198,18 +195,16 @@ func cleanTVSeasons(original, toRemove []types.TVSeasonResult) []types.TVSeasonR return cleaned } -func discBeatsPlexResolution(lowestPlexResolution string, format []string) bool { - for i := range format { - switch format[i] { - case types.Disk4K: - return true // 4K beats everything - case types.DiskBluray: - if slices.Contains([]string{types.PlexResolution1080, types.PlexResolution720, // HD - types.PlexResolution576, types.PlexResolution480, types.PlexResolution240, types.PlexResolutionSD}, // SD - lowestPlexResolution) { - return true - } - } // DVD is not considered - } +func discBeatsPlexResolution(lowestPlexResolution, format string) bool { + switch format { + case types.Disk4K: + return true // 4K beats everything + case types.DiskBluray: + if slices.Contains([]string{types.PlexResolution1080, types.PlexResolution720, // HD + types.PlexResolution576, types.PlexResolution480, types.PlexResolution240, types.PlexResolutionSD}, // SD + lowestPlexResolution) { + return true + } + } // DVD is not considered return false } diff --git a/web/tv/tv_test.go b/web/tv/tv_test.go index 7fbc5b7..6c2c9a4 100644 --- a/web/tv/tv_test.go +++ b/web/tv/tv_test.go @@ -41,7 +41,7 @@ func TestCleanTVSeries(t *testing.T) { func Test_discBeatsPlexResolution(t *testing.T) { type args struct { lowestResolution string - format []string + format string } tests := []struct { name string @@ -52,7 +52,7 @@ func Test_discBeatsPlexResolution(t *testing.T) { name: "4K disc beats everything", args: args{ lowestResolution: types.PlexResolution4K, - format: []string{types.Disk4K}, + format: types.Disk4K, }, want: true, }, @@ -60,7 +60,7 @@ func Test_discBeatsPlexResolution(t *testing.T) { name: "Blu-ray disc beats 1080p", args: args{ lowestResolution: types.PlexResolution1080, - format: []string{types.DiskBluray}, + format: types.DiskBluray, }, want: true, }, @@ -68,7 +68,7 @@ func Test_discBeatsPlexResolution(t *testing.T) { name: "Blu-ray disc is beaten by 4K", args: args{ lowestResolution: types.PlexResolution4K, - format: []string{types.DiskBluray}, + format: types.DiskBluray, }, want: false, },