Merge pull request #890 from AudricV/yt-clients-update-improvements-a…

…nd-extraction-fixes [YouTube] Update clients version and improve extraction of API key and client version of WEB client
TeamNewPipe · Aug 12, 2022 · 76aad92 · 76aad92
2 parents 6a885ef + 472f5d9
commit 76aad92
Show file tree

Hide file tree

Showing 268 changed files with 35,732 additions and 34,643 deletions.
diff --git a/...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
diff --git a/...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
@@ -932,6 +932,10 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
                 mobileBody, localization, "&t=" + generateTParameter()
                         + "&id=" + videoId);
 
+        if (isPlayerResponseNotValid(androidPlayerResponse, videoId)) {
+            return;
+        }
+
         final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
             androidStreamingData = streamingData;
@@ -963,6 +967,10 @@ private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCount
                 mobileBody, localization, "&t=" + generateTParameter()
                         + "&id=" + videoId);
 
+        if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
+            return;
+        }
+
         final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
             iosStreamingData = streamingData;
@@ -1001,6 +1009,38 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo
         }
     }
 
+    /**
+     * Checks whether an additional player response is not valid.
+     *
+     * <p>
+     * If YouTube detect that requests come from a third party client, they may replace the real
+     * player response by another one of a video saying that this content is not available on this
+     * app and to watch it on the latest version of YouTube.
+     * </p>
+     *
+     * <p>
+     * We can detect this by checking whether the video ID of the player response returned is the
+     * same as the one requested by the extractor.
+     * </p>
+     *
+     * <p>
+     * This behavior has been already observed on the {@code ANDROID} client, see
+     * <a href="https://github.com/TeamNewPipe/NewPipe/issues/8713">
+     *     https://github.com/TeamNewPipe/NewPipe/issues/8713</a>.
+     * </p>
+     *
+     * @param additionalPlayerResponse an additional response to the one of the {@code HTML5}
+     *                                 client used
+     * @param videoId                  the video ID of the content requested
+     * @return whether the video ID of the player response is not equal to the one requested
+     */
+    private static boolean isPlayerResponseNotValid(
+            @Nonnull final JsonObject additionalPlayerResponse,
+            @Nonnull final String videoId) {
+        return !videoId.equals(additionalPlayerResponse.getObject("videoDetails")
+                .getString("videoId", ""));
+    }
+
     private static void storePlayerJs() throws ParsingException {
         try {
             playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();

diff --git a/extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json b/extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json
@@ -3,14 +3,14 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Accept-Language": [
-        "en-GB, en;q\u003d0.9"
-      ],
       "Origin": [
         "https://www.youtube.com"
       ],
       "Referer": [
         "https://www.youtube.com"
+      ],
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
       ]
     },
     "localization": {
@@ -37,14 +37,17 @@
       "content-type": [
         "text/javascript; charset\u003dutf-8"
       ],
+      "critical-ch": [
+        "Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Model, Sec-CH-UA-Platform, Sec-CH-UA-Platform-Version"
+      ],
       "cross-origin-opener-policy-report-only": [
-        "same-origin; report-to\u003d\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\""
+        "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Tue, 15 Mar 2022 17:06:48 GMT"
+        "Fri, 12 Aug 2022 17:15:42 GMT"
       ],
       "expires": [
-        "Tue, 15 Mar 2022 17:06:48 GMT"
+        "Fri, 12 Aug 2022 17:15:42 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -53,15 +56,15 @@
         "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
       ],
       "report-to": [
-        "{\"group\":\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/encsid_ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\"}]}"
+        "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
       ],
       "server": [
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dk7SuAYyBwyE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 19-Jun-2019 17:06:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+647; expires\u003dThu, 14-Mar-2024 17:06:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003dzvoZRVLWdUQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 16-Nov-2019 17:15:42 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+621; expires\u003dSun, 11-Aug-2024 17:15:42 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -76,7 +79,7 @@
         "0"
       ]
     },
-    "responseBody": "/** 18082585275338869083 */self.document \u003d self; self.window \u003d self;var ytcfg\u003d{d:function(){return window.yt\u0026\u0026yt.config_||ytcfg.data_||(ytcfg.data_\u003d{})},get:function(k,o){return k in ytcfg.d()?ytcfg.d()[k]:o},set:function(){var a\u003darguments;if(a.length\u003e1)ytcfg.d()[a[0]]\u003da[1];else for(var k in a[0])ytcfg.d()[k]\u003da[0][k]}};\nytcfg.set({\"EXPERIMENT_FLAGS\":{\"allow_music_base_url\":true,\"allow_skip_networkless\":true,\"clear_user_partitioned_ls\":true,\"deprecate_two_way_binding_child\":true,\"deprecate_two_way_binding_parent\":true,\"desktop_image_cta_no_background\":true,\"desktop_text_ads_gray_visurl\":true,\"disable_child_node_auto_formatted_strings\":true,\"disable_simple_mixed_direction_formatted_strings\":true,\"disable_thumbnail_preloading\":true,\"enable_client_sli_logging\":true,\"enable_gel_log_commands\":true,\"enable_gray_visurl\":true,\"enable_mixed_direction_formatted_strings\":true,\"enable_server_stitched_dai\":true,\"enable_sli_flush\":true,\"enable_tectonic_ad_ux_for_halftime\":true,\"enable_topsoil_wta_for_halftime_live_infra\":true,\"enable_web_eom_visitor_data\":true,\"export_networkless_options\":true,\"forward_domain_admin_state_on_embeds\":true,\"html5_enable_single_video_vod_ivar_on_pacf\":true,\"html5_enable_video_overlay_on_inplayer_slot_for_tv\":true,\"html5_pacf_enable_dai\":true,\"html5_recognize_predict_start_cue_point\":true,\"html5_server_stitched_dai_group\":true,\"json_condensed_response\":true,\"kevlar_dropdown_fix\":true,\"kevlar_gel_error_routing\":true,\"log_heartbeat_with_lifecycles\":true,\"log_web_endpoint_to_layer\":true,\"networkless_gel\":true,\"networkless_logging\":true,\"nwl_send_fast_on_unload\":true,\"offline_error_handling\":true,\"pageid_as_header_web\":true,\"pes_migrate_association_data\":true,\"polymer_bad_build_labels\":true,\"polymer_verifiy_app_state\":true,\"qoe_send_and_write\":true,\"record_app_crashed_web\":true,\"search_image_menu_alignment\":true,\"skip_ls_gel_retry\":true,\"suppress_error_204_logging\":true,\"sw_nav_request_network_first\":true,\"use_bg_facade\":true,\"use_document_lifecycles\":true,\"use_screen_manager_util\":true,\"use_undefined_csn_any_layer\":true,\"vss_final_ping_send_and_write\":true,\"vss_playback_use_send_and_write\":true,\"web_api_url\":true,\"web_click_command_fallback\":true,\"web_dedupe_ve_grafting\":true,\"web_deprecate_service_ajax_map_dependency\":true,\"web_enable_ad_signals_in_it_context\":true,\"web_fetch_promise_cleanup_killswitch\":true,\"web_forward_command_on_pbj\":true,\"web_log_memory_total_kbytes\":true,\"web_ordered_response_processors\":true,\"web_prefetch_preload_video\":true,\"web_yt_config_context\":true,\"ytidb_fetch_datasync_ids_for_data_cleanup\":true,\"addto_ajax_log_warning_fraction\":0.1,\"log_window_onerror_fraction\":0.1,\"ytidb_transaction_ended_event_rate_limit\":0.02,\"botguard_async_snapshot_timeout_ms\":3000,\"check_navigator_accuracy_timeout_ms\":0,\"initial_gel_batch_timeout\":2000,\"network_polling_interval\":30000,\"polymer_log_prop_change_observer_percent\":0,\"web_foreground_heartbeat_interval_ms\":28000,\"web_logging_max_batch\":150,\"ytidb_remake_db_retries\":1,\"ytidb_reopen_db_retries\":0,\"cb_v2_uxe\":\"23983171\",\"web_client_version_override\":\"\",\"kevlar_command_handler_command_banlist\":[],\"web_op_continuation_type_banlist\":[],\"web_op_endpoint_banlist\":[],\"web_op_signal_type_banlist\":[]},\"INNERTUBE_API_KEY\":\"AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\",\"INNERTUBE_API_VERSION\":\"v1\",\"INNERTUBE_CLIENT_NAME\":\"WEB\",\"INNERTUBE_CLIENT_VERSION\":\"2.20220315.01.00\",\"INNERTUBE_CONTEXT\":{\"client\":{\"clientName\":\"WEB\",\"clientVersion\":\"2.20220315.01.00\"}},\"INNERTUBE_CONTEXT_CLIENT_NAME\":1,\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"2.20220315.01.00\",\"LATEST_ECATCHER_SERVICE_TRACKING_PARAMS\":{\"client.name\":\"WEB\"}});window.ytcfg.obfuscatedData_ \u003d [];if (self.trustedTypes \u0026\u0026 self.trustedTypes.createPolicy) {const swPolicy \u003d trustedTypes.createPolicy(\u0027youtubeServiceWorkerPolicy\u0027, {createScriptURL: function(ignored) { return \u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027; }}); importScripts(swPolicy.createScriptURL(\u0027\u0027));} else {importScripts(\u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027);}",
+    "responseBody": "\n  self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n    event.waitUntil(self.skipWaiting());\n  });\n  self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n    event.waitUntil(\n      self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n  });\n  ",
     "latestUrl": "https://www.youtube.com/sw.js"
   }
 }