Added way to test HTML & forcing all Instagram request with header

Udhayarajan · Jun 27, 2023 · 12a8ead · 12a8ead
1 parent cccfc89
commit 12a8ead
Show file tree

Hide file tree

Showing 14 changed files with 61 additions and 27 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -27,7 +27,7 @@ plugins {
 }
 
 group = "io.github.udhayarajan"
-version = "5.3.9"
+version = "5.3.10"
 //Version Naming incremented if "<NEW_FEATURE_ADDED>.<WORKED_ON_BUG>.<BETA_VERSION_COUNT>"
 //Priority on incrementing Feature > BugFix > Beta
 

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt
@@ -107,6 +107,10 @@ class DailyMotion(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractFromM3U8(response: String) {
         fun valueForKey(key: String, line: String): String? {
             val matcher = Pattern.compile("$key=(?:\"(.*?)\"|(.*?),)").matcher(line)

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt
@@ -119,11 +119,12 @@ abstract class Extractor(
 
     private suspend fun safeAnalyze() {
         try {
-            if (inputUrl.contains("instagram"))
+            if (inputUrl.contains("instagram")) {
                 inputUrl = if (cookies == null) {
                     inputUrl.replace("/reels/", "/reel/")
                 } else inputUrl.replace("/reel/", "/reels/")
-
+                headers["User-Agent"] = "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
+            }
             if (HttpRequest(inputUrl, headers).isAvailable())
                 analyze()
             else if (inputUrl.contains("instagram") && cookies != null) {
@@ -180,7 +181,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (videoData in format.videoData) {
-                sizes.add(async { HttpRequest(videoData.url).getSize() })
+                sizes.add(async { HttpRequest(videoData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -190,7 +191,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (audioData in format.audioData) {
-                sizes.add(async { HttpRequest(audioData.url).getSize() })
+                sizes.add(async { HttpRequest(audioData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -200,7 +201,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (imageData in format.imageData) {
-                sizes.add(async { HttpRequest(imageData.url).getSize() })
+                sizes.add(async { HttpRequest(imageData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -213,4 +214,6 @@ abstract class Extractor(
     protected inline fun internalError(msg: String, e: Exception? = null) {
         onProgress(Result.Failed(Error.InternalError(msg, e)))
     }
+
+    abstract suspend fun testWebpage(string: String)
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt
@@ -523,7 +523,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
         return null
     }
 
-    suspend fun testWebpage(string: String) {
+    override suspend fun testWebpage(string: String) {
         onProgress = {
 
         }

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt
@@ -156,14 +156,14 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 inputUrl = inputUrl.replace("https://instagram.com", "https://www.instagram.com")
                 logger.info("The new url is $inputUrl&__a=1&__d=dis")
                 val items =
-                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), getHeadersWithUserAgent()).getResponse(true) ?: run {
+                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), headers).getResponse(true) ?: run {
                         clientRequestError("unable to get post event with __a=1&__d=dis")
                         return
                     }
                 extractFromItems(items.toJSONObject().getJSONArray("items"))
                 return
             }
-            extractInfoShared(HttpRequest(inputUrl, getHeadersWithUserAgent()).getResponse() ?: run {
+            extractInfoShared(HttpRequest(inputUrl, headers).getResponse() ?: run {
                 clientRequestError()
                 return
             })
@@ -182,7 +182,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
     }
 
     private suspend fun extractHighlights(highlightsId: String) {
-        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), getHeadersWithUserAgent()).getResponse()
+        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), headers).getResponse()
             ?.toJSONObjectOrNull()
         highlights?.let {
             if (it.getNullable("login_required") == "true") {
@@ -197,8 +197,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
 
     @Suppress("UNCHECKED_CAST")
     private suspend fun extractStories(userId: String) {
-        val dupHeader = getHeadersWithUserAgent()
-        val stories = HttpRequest(STORIES_API.format(userId), dupHeader).getResponse()
+        val stories = HttpRequest(STORIES_API.format(userId), headers).getResponse()
         val reel = JSONObject(stories).getNullableJSONObject("reel")
         reel?.let { extractFromItems(it.getJSONArray("items")) } ?: onProgress(
             Result.Failed(
@@ -209,12 +208,6 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         )
     }
 
-    private fun getHeadersWithUserAgent(): Hashtable<String, String> {
-        val dupHeader: Hashtable<String, String> = headers.clone() as Hashtable<String, String>
-        dupHeader["User-Agent"] =
-            "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
-        return dupHeader
-    }
 
 
     private suspend fun extractInfoShared(page: String) {
@@ -228,7 +221,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 if (mediaId == null) throw JSONException("mediaId is null purposely thrown wrong error")
                 val url = POST_API.format(mediaId)
                 extractFromItems(
-                    HttpRequest(url, getHeadersWithUserAgent())
+                    HttpRequest(url, headers)
                         .getResponse()
                         .toString()
                         .toJSONObject()
@@ -378,7 +371,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         val matcher = Pattern.compile("<link rel=\"preload\" href=\"(.*?)\" as=\"script\"").matcher(page)
         while (matcher.find()) {
             ids.addAll(
-                getQueryHash(HttpRequest(matcher.group(1)).getResponse() ?: run {
+                getQueryHash(HttpRequest(matcher.group(1), headers).getResponse() ?: run {
                     return null
                 })
             )
@@ -560,4 +553,12 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         } else
             finalize()
     }
+
+    override suspend fun testWebpage(string: String) {
+        onProgress = {
+            println(it)
+        }
+        extractInfoShared(string)
+    }
+
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt
@@ -57,6 +57,10 @@ class Likee internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractVideoList(jsonArray: JSONArray) {
         for (i in 0 until jsonArray.length()) {
             val localFormats = formats.copy(title = "", videoData = mutableListOf(), imageData = mutableListOf())

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt
@@ -42,6 +42,10 @@ class LinkedIn internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebpage(page: String) {
         val matcher = Pattern.compile("data-sources=\"(.*?)\"").matcher(page)
         if (matcher.find()) {

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt
@@ -43,6 +43,10 @@ class Periscope internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     var manifest: ArrayList<ArrayList<String>>? = ArrayList()
 
 

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt
@@ -42,6 +42,10 @@ class ShareChat internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebPage(response: String) {
         val matcher =
             Pattern.compile("""<script data-rh="true" type="application\/ld\+json">(\{"@context":"http:\/\/schema\.org","@type":"(?:Image|Video)Object".*?\})<\/script>""")

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt
@@ -80,7 +80,7 @@ class TikTok internal constructor(url: String) : Extractor(url) {
             onProgress(Result.Failed(Error.MethodMissingLogic))
     }
 
-    suspend fun testWithWebPage(string: String){
+    override suspend fun testWebpage(string: String) {
         onProgress = {
             println(it)
         }

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt
@@ -38,6 +38,10 @@ class Twitch internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private fun getId(s: String?): String? {
         return s?.run {
             val matcher =

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt
@@ -235,6 +235,10 @@ class Twitter internal constructor(url: String) : Extractor(url) {
         super.finalize()
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
 
     fun resolution(url: String?): String {
         val pattern: Pattern = Pattern.compile("/(\\d+x\\d+)/")

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt
@@ -64,6 +64,10 @@ class Vimeo internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun parseConfigRequest(response: String) {
         val json = response.toJSONObject()
         val hls = json.getJSONObject("request").getJSONObject("files").getJSONObject("dash")

diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt
@@ -42,13 +42,11 @@ class HttpRequest(
     companion object {
         private var prefixUrl = ""
         private var additionHeader: Hashtable<String, String>? = null
-        private fun defaultClient(requiresRedirection: Boolean = true) = HttpClient(Android).also {
-            it.config {
+        private fun defaultClient(requiresRedirection: Boolean = true) = HttpInterfaceImpl(
+            HttpClient(Android).config {
                 followRedirects = requiresRedirection
             }
-        }.run {
-            HttpInterfaceImpl(this)
-        }
+        )
 
         private var clientGenerator: () -> HttpClient = {
             HttpClient(Android)
@@ -104,7 +102,7 @@ class HttpRequest(
      * @return bytes count of given [url]
      */
     suspend fun getSize(useCustomClient: Boolean = true) =
-        (if (useCustomClient) createClient() else defaultClient()).getSize(url)
+        (if (useCustomClient) createClient() else defaultClient()).getSize(url, getHeader())
 
     suspend fun postRequest(postData: Hashtable<String, Any>? = null, useCustomClient: Boolean = true): String =
         withContext(Dispatchers.IO) {