Fix all tests

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -27,7 +27,6 @@
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
 import static java.util.Collections.singletonList;
 
 import com.grack.nanojson.JsonArray;
@@ -234,36 +233,31 @@ private YoutubeParsingHelper() {
 
     private static Random numberGenerator = new SecureRandom();
 
+    private static final String FEED_BASE_CHANNEL_ID =
+            "https://www.youtube.com/feeds/videos.xml?channel_id=";
+    private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
+    private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
+    private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
+            Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
+    private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
+    private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
+
     /**
-     * {@code PENDING+} means that the user did not yet submit their choices.
+     * Determines how the consent cookie (that is required for YouTube) will be generated.
      *
      * <p>
-     * Therefore, YouTube & Google should not track the user, because they did not give consent.
+     * {@code false} (default) will use {@code PENDING+}.
+     * {@code true} will use {@code YES+}.
      * </p>
      *
      * <p>
-     * The three digits at the end can be random, but are required.
+     * Setting this value to <code>true</code> is currently needed if you want to watch
+     * Mix Playlists in some countries (EU).
      * </p>
-     */
-    private static final String CONSENT_COOKIE_VALUE = "PENDING+";
-
-    /**
-     * YouTube {@code CONSENT} cookie.
      *
-     * <p>
-     * Should prevent redirect to {@code consent.youtube.com}.
-     * </p>
+     * @see #generateConsentCookie()
      */
-    private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
-
-    private static final String FEED_BASE_CHANNEL_ID =
-            "https://www.youtube.com/feeds/videos.xml?channel_id=";
-    private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
-    private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
-    private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
-            Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
-    private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
-    private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
+    private static boolean consentAccepted = false;
 
     private static boolean isGoogleURL(final String url) {
         final String cachedUrl = extractCachedUrlIfNeeded(url);
@@ -1378,7 +1372,6 @@ public static Map<String, List<String>> getCookieHeader() {
 
     /**
      * Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
-     * @see #CONSENT_COOKIE
      * @param headers the headers which should be completed
      */
     public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@@ -1391,8 +1384,13 @@ public static void addCookieHeader(@Nonnull final Map<String, List<String>> head
 
     @Nonnull
     public static String generateConsentCookie() {
-        final int statusCode = 100 + numberGenerator.nextInt(900);
-        return CONSENT_COOKIE + statusCode;
+        return "CONSENT=" + (isConsentAccepted()
+                // YES+ means that the user did submit their choices and allows tracking.
+                ? "YES+"
+                // PENDING+ means that the user did not yet submit their choices.
+                // YT & Google should not track the user, because they did not give consent.
+                // The three digits at the end can be random, but are required.
+                : "PENDING+" + (100 + numberGenerator.nextInt(900)));
     }
 
     public static String extractCookieValue(final String cookieName,
@@ -1612,16 +1610,6 @@ public static boolean isVerified(final JsonArray badges) {
         return false;
     }
 
-    @Nonnull
-    public static String unescapeDocument(@Nonnull final String doc) {
-        return doc
-                .replaceAll("\\\\x22", "\"")
-                .replaceAll("\\\\x7b", "{")
-                .replaceAll("\\\\x7d", "}")
-                .replaceAll("\\\\x5b", "[")
-                .replaceAll("\\\\x5d", "]");
-    }
-
     /**
      * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
      * playback requests (and also for some clients, in the player request body).
@@ -1692,4 +1680,18 @@ public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
     public static boolean isIosStreamingUrl(@Nonnull final String url) {
         return Parser.isMatch(C_IOS_PATTERN, url);
     }
+
+    /**
+     * @see #consentAccepted
+     */
+    public static void setConsentAccepted(final boolean accepted) {
+        consentAccepted = accepted;
+    }
+
+    /**
+     * @see #consentAccepted
+     */
+    public static boolean isConsentAccepted() {
+        return consentAccepted;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java

@@ -5,12 +5,13 @@
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.StringUtils;
 
-import javax.annotation.Nonnull;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nonnull;
+
 /**
  * YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their
  * {@code n} query parameter.
@@ -154,16 +155,25 @@ private static String parseDecodeFunction(final String playerJsCode, final Strin
     private static String parseWithParenthesisMatching(final String playerJsCode,
                                                        final String functionName) {
         final String functionBase = functionName + "=function";
-        return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
-                + ";";
+        return validateFunction(functionBase
+                + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
+                + ";");
     }
 
     @Nonnull
     private static String parseWithRegex(final String playerJsCode, final String functionName)
             throws Parser.RegexException {
-        final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
+        final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?};)\n",
                 Pattern.DOTALL);
-        return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
+        return validateFunction("function "
+                + functionName
+                + Parser.matchGroup1(functionPattern, playerJsCode));
+    }
+
+    @Nonnull
+    private static String validateFunction(@Nonnull final String function) {
+        JavaScript.compileOrThrow(function);
+        return function;
     }
 
     @Deprecated

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -2,7 +2,7 @@
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
@@ -23,6 +23,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -89,16 +90,26 @@ public void onFetchPage(@Nonnull final Downloader downloader)
         final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
 
         final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
+        // Cookie is required due to consent
+        addYouTubeHeaders(headers);
 
         final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
                 + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
 
         initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
-        playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
-                .getObject("playlist").getObject("playlist");
+        playlistData = initialData
+                .getObject("contents")
+                .getObject("twoColumnWatchNextResults")
+                .getObject("playlist")
+                .getObject("playlist");
         if (isNullOrEmpty(playlistData)) {
-            throw new ExtractionException("Could not get playlistData");
+            final ExtractionException ex = new ExtractionException("Could not get playlistData");
+            if (!YoutubeParsingHelper.isConsentAccepted()) {
+                throw new ContentNotAvailableException(
+                        "Consent is required in some countries to view Mix playlists",
+                        ex);
+            }
+            throw ex;
         }
         cookieValue = extractCookieValue(COOKIE_NAME, response);
     }
@@ -212,7 +223,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
 
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
+        // Cookie is required due to consent
+        addYouTubeHeaders(headers);
 
         final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
                 getExtractorLocalization());

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -22,6 +22,7 @@
 
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.MetaInfo;
+import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -31,7 +32,6 @@
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
-import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
@@ -43,6 +43,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -133,55 +134,52 @@ public void onFetchPage(@Nonnull final Downloader downloader)
         }
     }
 
-    @Nonnull
-    @Override
-    public String getUrl() throws ParsingException {
-        return super.getUrl();
+    private List<JsonObject> getItemSectionRendererContents() {
+        return initialData
+                .getObject("contents")
+                .getObject("tabbedSearchResultsRenderer")
+                .getArray("tabs")
+                .getObject(0)
+                .getObject("tabRenderer")
+                .getObject("content")
+                .getObject("sectionListRenderer")
+                .getArray("contents")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(c -> c.getObject("itemSectionRenderer"))
+                .filter(isr -> !isr.isEmpty())
+                .map(isr -> isr
+                        .getArray("contents")
+                        .getObject(0))
+                .collect(Collectors.toList());
     }
 
     @Nonnull
     @Override
     public String getSearchSuggestion() throws ParsingException {
-        final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
-                "contents.tabbedSearchResultsRenderer.tabs").getObject(0),
-                "tabRenderer.content.sectionListRenderer.contents")
-                .getObject(0)
-                .getObject("itemSectionRenderer");
-        if (itemSectionRenderer.isEmpty()) {
-            return "";
+        for (final JsonObject obj : getItemSectionRendererContents()) {
+            final JsonObject didYouMeanRenderer = obj
+                    .getObject("didYouMeanRenderer");
+            final JsonObject showingResultsForRenderer = obj
+                    .getObject("showingResultsForRenderer");
+
+            if (!didYouMeanRenderer.isEmpty()) {
+                return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
+            } else if (!showingResultsForRenderer.isEmpty()) {
+                return JsonUtils.getString(showingResultsForRenderer,
+                        "correctedQueryEndpoint.searchEndpoint.query");
+            }
         }
 
-        final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
-                .getObject(0).getObject("didYouMeanRenderer");
-        final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
-                .getObject(0)
-                .getObject("showingResultsForRenderer");
-
-        if (!didYouMeanRenderer.isEmpty()) {
-            return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
-        } else if (!showingResultsForRenderer.isEmpty()) {
-            return JsonUtils.getString(showingResultsForRenderer,
-                    "correctedQueryEndpoint.searchEndpoint.query");
-        } else {
-            return "";
-        }
+        return "";
     }
 
     @Override
     public boolean isCorrectedSearch() throws ParsingException {
-        final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
-                "contents.tabbedSearchResultsRenderer.tabs").getObject(0),
-                "tabRenderer.content.sectionListRenderer.contents")
-                .getObject(0)
-                .getObject("itemSectionRenderer");
-        if (itemSectionRenderer.isEmpty()) {
-            return false;
-        }
-
-        final JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
-
-        return firstContent.has("didYouMeanRenderer")
-                || firstContent.has("showingResultsForRenderer");
+        return getItemSectionRendererContents()
+                .stream()
+                .anyMatch(obj -> obj.has("showingResultsForRenderer"));
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java

@@ -9,6 +9,18 @@ public final class JavaScript {
     private JavaScript() {
     }
 
+    public static void compileOrThrow(final String function) {
+        try {
+            final Context context = Context.enter();
+            context.setOptimizationLevel(-1);
+
+            // If it doesn't compile it throws an exception here
+            context.compileString(function, null, 1, null);
+        } finally {
+            Context.exit();
+        }
+    }
+
     public static String run(final String function,
                              final String functionName,
                              final String... parameters) {