[YouTube] Workaround HTTP 403s on streaming URLs of WEB client (after some time or instantly for some JavaScript players), update clients info

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -152,7 +152,7 @@ private YoutubeParsingHelper() {
      * The client version for InnerTube requests with the {@code WEB} client, used as the last
      * fallback if the extraction of the real one failed.
      */
-    private static final String HARDCODED_CLIENT_VERSION = "2.20240410.01.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00";
 
     /**
      * The hardcoded client version of the Android app used for InnerTube requests with this
@@ -163,7 +163,7 @@ private YoutubeParsingHelper() {
      * such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
      * </p>
      */
-    private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.13.36";
+    private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35";
 
     /**
      * The hardcoded client version of the iOS app used for InnerTube requests with this client.
@@ -174,7 +174,7 @@ private YoutubeParsingHelper() {
      * Store page of the YouTube app</a>, in the {@code What’s New} section.
      * </p>
      */
-    private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.14.3";
+    private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";
 
     /**
      * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
@@ -190,7 +190,7 @@ private YoutubeParsingHelper() {
      * The hardcoded client version used for InnerTube requests with the YouTube Music desktop
      * client.
      */
-    private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240403.01.00";
+    private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00";
 
     private static String clientVersion;
 
@@ -219,31 +219,31 @@ private YoutubeParsingHelper() {
      * information.
      * </p>
      */
-    private static final String IOS_DEVICE_MODEL = "iPhone15,4";
+    private static final String IOS_DEVICE_MODEL = "iPhone16,2";
 
     /**
-     * Spoofing an iPhone 15 running iOS 17.4.1 with the hardcoded version of the iOS app. To be
-     * used for the {@code "osVersion"} field in JSON POST requests.
+     * Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app.
+     * To be used for the {@code "osVersion"} field in JSON POST requests.
      * <p>
      * The value of this field seems to use the following structure:
      * "iOS major version.minor version.patch version.build version", where
      * "patch version" is equal to 0 if it isn't set
      * The build version corresponding to the iOS version used can be found on
-     * <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15">
-     *     https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15</a>
+     * <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max">
+     *     https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max</a>
      * </p>
      *
      * @see #IOS_USER_AGENT_VERSION
      */
-    private static final String IOS_OS_VERSION = "17.4.1.21E237";
+    private static final String IOS_OS_VERSION = "17.5.1.21F90";
 
     /**
-     * Spoofing an iPhone 15 running iOS 17.4.1 with the hardcoded version of the iOS app. To be
+     * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be
      * used in the user agent for requests.
      *
      * @see #IOS_OS_VERSION
      */
-    private static final String IOS_USER_AGENT_VERSION = "17_4_1";
+    private static final String IOS_USER_AGENT_VERSION = "17_5_1";
 
     private static Random numberGenerator = new Random();
 
@@ -1301,31 +1301,49 @@ public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
     }
 
     @Nonnull
-    public static byte[] createDesktopPlayerBody(
+    public static JsonObject getWebPlayerResponse(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId) throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(
+                        prepareDesktopJsonBuilder(localization, contentCountry)
+                                .value(VIDEO_ID, videoId)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+        final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+                + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(
+                        url, getYouTubeHeaders(), body, localization)));
+    }
+
+    @Nonnull
+    public static byte[] createTvHtml5EmbedPlayerBody(
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
             @Nonnull final String videoId,
             @Nonnull final Integer sts,
-            final boolean isTvHtml5DesktopJsonBuilder,
-            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
+            @Nonnull final String contentPlaybackNonce) {
         // @formatter:off
-        return JsonWriter.string((isTvHtml5DesktopJsonBuilder
-                        ? prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
-                        : prepareDesktopJsonBuilder(localization, contentCountry))
-                .object("playbackContext")
-                    .object("contentPlaybackContext")
-                        // Signature timestamp from the JavaScript base player is needed to get
-                        // working obfuscated URLs
-                        .value("signatureTimestamp", sts)
-                        .value("referer", "https://www.youtube.com/watch?v=" + videoId)
+        return JsonWriter.string(
+                prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
+                    .object("playbackContext")
+                        .object("contentPlaybackContext")
+                            // Signature timestamp from the JavaScript base player is needed to get
+                            // working obfuscated URLs
+                            .value("signatureTimestamp", sts)
+                            .value("referer", "https://www.youtube.com/watch?v=" + videoId)
+                        .end()
                     .end()
-                .end()
-                .value(CPN, contentPlaybackNonce)
-                .value(VIDEO_ID, videoId)
-                .value(CONTENT_CHECK_OK, true)
-                .value(RACY_CHECK_OK, true)
-                .done())
-                .getBytes(StandardCharsets.UTF_8);
+                    .value(CPN, contentPlaybackNonce)
+                    .value(VIDEO_ID, videoId)
+                    .value(CONTENT_CHECK_OK, true)
+                    .value(RACY_CHECK_OK, true)
+                    .done())
+                    .getBytes(StandardCharsets.UTF_8);
         // @formatter:on
     }
 
@@ -1366,7 +1384,7 @@ public static String getAndroidUserAgent(@Nullable final Localization localizati
      */
     @Nonnull
     public static String getIosUserAgent(@Nullable final Localization localization) {
-        // Spoofing an iPhone 15 running iOS 17.4.1 with the hardcoded version of the iOS app
+        // Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app
         return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
                 + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
                 + IOS_USER_AGENT_VERSION + " like Mac OS X; "

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java

@@ -26,24 +26,54 @@ final class YoutubeThrottlingParameterUtils {
 
     private static final String ARRAY_ACCESS_REGEX = "\\[(\\d+)]";
 
-    /**
-     * The first regex matches this, where we want BDa:
-     * <p>
-     * (b=String.fromCharCode(110),c=a.get(b))&&(c=<strong>BDa</strong><strong>[0]</strong>(c)
-     * <p>
-     * Array access is optional, but needs to be handled, since the actual function is inside the
-     * array.
-     */
     // CHECKSTYLE:OFF
     private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = {
+
+            /*
+             * The first regex matches the following text, where we want rDa and the array index
+             * accessed:
+             *
+             * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "+=\"nn\"\\[\\+"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+\\." + SINGLE_CHAR_VARIABLE_REGEX + "+],"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+=" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "+\\.get\\(" + SINGLE_CHAR_VARIABLE_REGEX + "+\\)\\)&&\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+=(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "+)\\[(\\d+)]"),
+
+            /*
+             * The second regex matches the following text, where we want rma:
+             *
+             * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "+=\"nn\"\\[\\+"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+\\." + SINGLE_CHAR_VARIABLE_REGEX + "+],"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+=" + SINGLE_CHAR_VARIABLE_REGEX + "+\\.get\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "+\\)\\).+\\|\\|(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "+)\\(\"\"\\)"),
+
+            /*
+             * The third regex matches the following text, where we want BDa and the array index
+             * accessed:
+             *
+             * (b=String.fromCharCode(110),c=a.get(b))&&(c=BDa[0](c)
+             */
             Pattern.compile("\\(" + SINGLE_CHAR_VARIABLE_REGEX + "=String\\.fromCharCode\\(110\\),"
                     + SINGLE_CHAR_VARIABLE_REGEX + "=" + SINGLE_CHAR_VARIABLE_REGEX + "\\.get\\("
                     + SINGLE_CHAR_VARIABLE_REGEX + "\\)\\)" + "&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
                     + "=(" + FUNCTION_NAME_REGEX + ")" + "(?:" + ARRAY_ACCESS_REGEX + ")?\\("
                     + SINGLE_CHAR_VARIABLE_REGEX + "\\)"),
+
+            /*
+             * The fourth regex matches the following text, where we want Yva and the array index
+             * accessed:
+             *
+             * .get("n"))&&(b=Yva[0](b)
+             */
             Pattern.compile("\\.get\\(\"n\"\\)\\)&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
                     + "=(" + FUNCTION_NAME_REGEX + ")(?:" + ARRAY_ACCESS_REGEX + ")?\\("
-                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)"),
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)")
     };
     // CHECKSTYLE:ON