[YouTube] Set EU consent cookie

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Request.java

@@ -123,12 +123,10 @@ public Builder url(String url) {
          * Any default headers that the implementation may have, <b>should</b> be overridden by these.
          */
         public Builder headers(@Nullable Map<String, List<String>> headers) {
-            if (headers == null) {
-                this.headers.clear();
-                return this;
-            }
             this.headers.clear();
-            this.headers.putAll(headers);
+            if (headers != null) {
+                this.headers.putAll(headers);
+            }
             return this;
         }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -28,12 +28,7 @@
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeParseException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -79,6 +74,19 @@ private YoutubeParsingHelper() {
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
     private static String[] youtubeMusicKeys;
 
+    private static Random numberGenerator = new Random();
+
+    /**
+     * <code>PENDING+</code> means that the user did not yet submit their choices.
+     * Therefore, YouTube & Google should not track the user, because they did not give consent.
+     * The three digits at the end can be random, but are required.
+     */
+    public static final String CONSENT_COOKIE_VALUE = "PENDING+";
+    /**
+     * Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
+     */
+    public static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
+
     private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
 
@@ -388,6 +396,15 @@ public static void resetClientVersionAndKey() {
         key = null;
     }
 
+    /**
+     * <p>
+     * <b>Only use in tests.</b>
+     * </p>
+     */
+    public static void setNumberGenerator(Random random) {
+        numberGenerator = random;
+    }
+
     public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
         final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
 
@@ -427,6 +444,7 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
         headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
         headers.put("Referer", Collections.singletonList("music.youtube.com"));
         headers.put("Content-Type", Collections.singletonList("application/json"));
+        addCookieHeader(headers);
 
         final String response = getDownloader().post(url, headers, json).responseBody();
 
@@ -629,34 +647,19 @@ public static String getValidJsonResponseBody(final Response response)
     public static Response getResponse(final String url, final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        addYouTubeHeaders(headers);
 
         final Response response = getDownloader().get(url, headers, localization);
         getValidJsonResponseBody(response);
 
         return response;
     }
 
-    public static String extractCookieValue(final String cookieName, final Response response) {
-        final List<String> cookies = response.responseHeaders().get("set-cookie");
-        int startIndex;
-        String result = "";
-        for (final String cookie : cookies) {
-            startIndex = cookie.indexOf(cookieName);
-            if (startIndex != -1) {
-                result = cookie.substring(startIndex + cookieName.length() + "=".length(),
-                        cookie.indexOf(";", startIndex));
-            }
-        }
-        return result;
-    }
-
     public static JsonArray getJsonResponse(final String url, final Localization localization)
             throws IOException, ExtractionException {
         Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        addYouTubeHeaders(headers);
+
         final Response response = getDownloader().get(url, headers, localization);
 
         return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
@@ -665,17 +668,69 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
     public static JsonArray getJsonResponse(final Page page, final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        if (!isNullOrEmpty(page.getCookies())) {
-            headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
-        }
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        addYouTubeHeaders(headers);
 
         final Response response = getDownloader().get(page.getUrl(), headers, localization);
 
         return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
     }
 
+    /**
+     * Add required headers and cookies to an existing headers Map.
+     * @see #addClientInfoHeaders(Map)
+     * @see #addCookieHeader(Map)
+     */
+    public static void addYouTubeHeaders(final Map<String, List<String>> headers)
+            throws IOException, ExtractionException {
+        addClientInfoHeaders(headers);
+        addCookieHeader(headers);
+    }
+
+    /**
+     * Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
+     * @param headers The headers which should be completed
+     */
+    public static void addClientInfoHeaders(final Map<String, List<String>> headers)
+            throws IOException, ExtractionException {
+        if (headers.get("X-YouTube-Client-Name") == null) {
+            headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        }
+        if (headers.get("X-YouTube-Client-Version") == null) {
+            headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
+        }
+    }
+
+    /**
+     * Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
+     * @see #CONSENT_COOKIE
+     * @param headers the headers which should be completed
+     */
+    public static void addCookieHeader(final Map<String, List<String>> headers) {
+        if (headers.get("Cookie") == null) {
+            headers.put("Cookie", Arrays.asList(generateConsentCookie()));
+        } else {
+            headers.get("Cookie").add(generateConsentCookie());
+        }
+    }
+
+    public static String generateConsentCookie() {
+        return CONSENT_COOKIE + 100 + numberGenerator.nextInt(900);
+    }
+
+    public static String extractCookieValue(final String cookieName, final Response response) {
+        final List<String> cookies = response.responseHeaders().get("set-cookie");
+        int startIndex;
+        String result = "";
+        for (final String cookie : cookies) {
+            startIndex = cookie.indexOf(cookieName);
+            if (startIndex != -1) {
+                result = cookie.substring(startIndex + cookieName.length() + "=".length(),
+                        cookie.indexOf(";", startIndex));
+            }
+        }
+        return result;
+    }
+
     /**
      * Shared alert detection function, multiple endpoints return the error similarly structured.
      * <p>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -20,7 +20,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -130,8 +132,11 @@ public long getStreamCount() {
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         collectStreamsFrom(collector, playlistData.getArray("contents"));
-        return new InfoItemsPage<>(collector,
-                new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
+
+        final Map<String, String> cookies = new HashMap<>();
+        cookies.put(COOKIE_NAME, cookieValue);
+
+        return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
     }
 
     private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -12,9 +12,10 @@
 
 import java.io.IOException;
 import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONSENT_COOKIE_VALUE;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
 /*
@@ -45,17 +46,20 @@ public YoutubeSuggestionExtractor(StreamingService service) {
 
     @Override
     public List<String> suggestionList(String query) throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
-        List<String> suggestions = new ArrayList<>();
+        final Downloader dl = NewPipe.getDownloader();
+        final List<String> suggestions = new ArrayList<>();
 
-        String url = "https://suggestqueries.google.com/complete/search"
+        final String url = "https://suggestqueries.google.com/complete/search"
                 + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
                 + "&jsonp=" + "JP"
                 + "&ds=" + "yt"
                 + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
                 + "&q=" + URLEncoder.encode(query, UTF_8);
 
-        String response = dl.get(url, getExtractorLocalization()).responseBody();
+        final Map<String, List<String>> headers = new HashMap<>();
+        addCookieHeader(headers);
+
+        String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
         // trim JSONP part "JP(...)"
         response = response.substring(3, response.length() - 1);
         try {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -13,6 +13,7 @@
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
 import java.io.IOException;
+import java.util.Random;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -32,6 +33,7 @@ public static class NotAvailable {
         @BeforeClass
         public static void setUp() throws IOException {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -54,6 +56,7 @@ public static class NotSupported {
         @BeforeClass
         public static void setUp() throws IOException {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
         }
 
@@ -71,6 +74,7 @@ public static class Gronkh implements BaseChannelExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -168,6 +172,7 @@ public static class VSauce implements BaseChannelExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -265,6 +270,7 @@ public static class Kurzgesagt implements BaseChannelExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -383,6 +389,7 @@ public static class CaptainDisillusion implements BaseChannelExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -478,6 +485,7 @@ public static class RandomChannel implements BaseChannelExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
             extractor = (YoutubeChannelExtractor) YouTube
                     .getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -15,6 +15,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -36,6 +37,7 @@ public static class Thomas {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -124,6 +126,7 @@ public static class EmptyComment {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -163,6 +166,7 @@ public static class HeartedByCreator {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);
@@ -205,6 +209,7 @@ public static class Pinned {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
             extractor = (YoutubeCommentsExtractor) YouTube
                     .getCommentsExtractor(url);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -8,6 +8,8 @@
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 
+import java.util.Random;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -24,6 +26,7 @@ public static class Kurzgesagt implements BaseListExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
             extractor = (YoutubeFeedExtractor) YouTube
                     .getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java

@@ -8,6 +8,8 @@
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
 
+import java.util.Random;
+
 import static org.junit.Assert.assertEquals;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
@@ -23,6 +25,7 @@ public static class Trending implements BaseListExtractorTest {
         @BeforeClass
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "trending"));
             extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor();
             extractor.fetchPage();