Update client version, fix some tests, update mocks and do some impro…

…vements

Add the origin and the referer headers with the https://www.youtube.com value for YouTube JSON POST requests.
Don't add the consent cookie header for the requests which use the youtubei/innertube API because it's uneeded.
Fix some tests and update YouTube mocks

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -66,7 +66,7 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
-    private static final String HARDCODED_CLIENT_VERSION = "2.20210413.07.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20210420.07.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
     private static String clientVersion;
     private static String key;
@@ -302,7 +302,6 @@ public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
         final Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
         headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_CLIENT_VERSION));
-        addCookieHeader(headers);
 
         // This endpoint is fetched by the YouTube website to get the items of its main menu and is
         // pretty lightweight (around 30kB)
@@ -322,7 +321,9 @@ private static void extractClientVersionAndKey() throws IOException, ExtractionE
         if (!keyAndVersionExtracted) return;
         // Don't provide a search term in order to have a smaller response
         final String url = "https://www.youtube.com/results?search_query=";
-        final String html = getDownloader().get(url).responseBody();
+        final Map<String, List<String>> headers = new HashMap<>();
+        addCookieHeader(headers);
+        final String html = getDownloader().get(url, headers).responseBody();
         final JsonObject initialData = getInitialData(html);
         final JsonArray serviceTrackingParams = initialData.getObject("responseContext")
                 .getArray("serviceTrackingParams");
@@ -477,7 +478,6 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
         headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
         headers.put("Referer", Collections.singletonList("music.youtube.com"));
         headers.put("Content-Type", Collections.singletonList("application/json"));
-        addCookieHeader(headers);
 
         final Response response = getDownloader().post(url, headers, json);
         final String responseBody = response.responseBody();
@@ -491,7 +491,9 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
         if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS;
 
         final String url = "https://music.youtube.com/";
-        final String html = getDownloader().get(url).responseBody();
+        final Map<String, List<String>> headers = new HashMap<>();
+        addCookieHeader(headers);
+        final String html = getDownloader().get(url, headers).responseBody();
 
         String key;
         try {
@@ -695,7 +697,7 @@ public static JsonObject getJsonPostResponse(final String endpoint,
                                                  final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        addYouTubeHeaders(headers);
+        addClientInfoHeaders(headers);
 
         final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/"
                 + endpoint + "?key=" + getKey(), headers, body, localization);
@@ -750,11 +752,18 @@ public static void addYouTubeHeaders(final Map<String, List<String>> headers)
     }
 
     /**
-     * Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
+     * Add the <code>X-YouTube-Client-Name</code>, <code>X-YouTube-Client-Version</code>,
+     * <code>Origin</code>, and <code>Referer</code> headers.
      * @param headers The headers which should be completed
      */
     public static void addClientInfoHeaders(final Map<String, List<String>> headers)
             throws IOException, ExtractionException {
+        if (headers.get("Origin") == null) {
+            headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+        }
+        if (headers.get("Referer") == null) {
+            headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
+        }
         if (headers.get("X-YouTube-Client-Name") == null) {
             headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -199,9 +199,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
 
         if (contents.getObject(0).has("playlistSegmentRenderer")) {
             for (final Object segment : contents) {
-                if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
-                    collectTrailerFrom(collector, ((JsonObject) segment));
-                } else if (((JsonObject) segment).getObject("playlistSegmentRenderer")
+                if (((JsonObject) segment).getObject("playlistSegmentRenderer")
                         .has("videoList")) {
                     collectStreamsFrom(collector, ((JsonObject) segment)
                             .getObject("playlistSegmentRenderer").getObject("videoList")
@@ -287,83 +285,4 @@ public long getViewCount() {
             }
         }
     }
-
-    private void collectTrailerFrom(final StreamInfoItemsCollector collector,
-                                    final JsonObject segment) {
-        collector.commit(new StreamInfoItemExtractor() {
-            @Override
-            public String getName() throws ParsingException {
-                return getTextFromObject(segment.getObject("playlistSegmentRenderer")
-                        .getObject("title"));
-            }
-
-            @Override
-            public String getUrl() throws ParsingException {
-                return YoutubeStreamLinkHandlerFactory.getInstance()
-                        .fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
-                                .getObject("playlistVideoPlayerRenderer").getString("videoId"))
-                        .getUrl();
-            }
-
-            @Override
-            public String getThumbnailUrl() {
-                return "";
-                /*final JsonArray thumbnails = initialAjaxJson.getObject(1)
-                        .getObject("playerResponse")
-                        .getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
-                // the last thumbnail is the one with the highest resolution
-                final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
-                return fixThumbnailUrl(url);*/
-            }
-
-            @Override
-            public StreamType getStreamType() {
-                return StreamType.VIDEO_STREAM;
-            }
-
-            @Override
-            public boolean isAd() {
-                return false;
-            }
-
-            @Override
-            public long getDuration() throws ParsingException {
-                return YoutubeParsingHelper.parseDurationString(
-                        getTextFromObject(segment.getObject("playlistSegmentRenderer")
-                                .getObject("segmentAnnotation")).split("•")[0]);
-            }
-
-            @Override
-            public long getViewCount() {
-                return -1;
-            }
-
-            @Override
-            public String getUploaderName() throws ParsingException {
-                return YoutubePlaylistExtractor.this.getUploaderName();
-            }
-
-            @Override
-            public String getUploaderUrl() throws ParsingException {
-                return YoutubePlaylistExtractor.this.getUploaderUrl();
-            }
-
-            @Override
-            public boolean isUploaderVerified() {
-                return false;
-            }
-
-            @Nullable
-            @Override
-            public String getTextualUploadDate() {
-                return null;
-            }
-
-            @Nullable
-            @Override
-            public DateWrapper getUploadDate() {
-                return null;
-            }
-        });
-    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -101,14 +101,14 @@ public void getContinuations() throws Exception {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
                 for (final StreamInfoItem item : streams.getItems()) {
-//                    TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // TODO Duplicates are appearing
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 
@@ -178,13 +178,13 @@ public void getContinuations() throws Exception {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
                 for (final StreamInfoItem item : streams.getItems()) {
-//                    TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // TODO Duplicates are appearing
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 
@@ -254,14 +254,14 @@ public void getContinuations() throws Exception {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
                 for (final StreamInfoItem item : streams.getItems()) {
                     // TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -54,7 +54,8 @@ public static void setUp() throws Exception {
     @Override public long expectedDislikeCountAtLeast() { return 38000; }
     @Override public boolean expectedHasRelatedItems() { return false; } // no related videos (!)
     @Override public int expectedAgeLimit() { return 18; }
-    @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
+    // Broken, video is available with the embedded player
+    // @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
     @Override public boolean expectedHasSubtitles() { return false; }
 
     @Override public String expectedCategory() {return "Entertainment"; }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -13,6 +13,7 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 
 import javax.annotation.Nullable;
 
@@ -22,7 +23,6 @@
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
 
-@Ignore("Video is not available in specific countries. Someone else has to generate mocks")
 public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "T4XJQO3qol8";
@@ -32,6 +32,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @BeforeClass
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -37,7 +37,6 @@ public static void setUp() throws Exception {
 
     @Override
     @Test
-    @Ignore("When visiting website it shows 'Lofi Girl', unknown why it's different in tests")
     public void testUploaderName() throws Exception {
         super.testUploaderName();
     }