From baa11e20bc899e8b28d05f7a544caedcda0aac5e Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 5 Aug 2023 23:03:17 -0400 Subject: [PATCH 1/9] Use Selenium as a WebView in the browser --- server/build.gradle.kts | 2 + .../suwayomi/tachidesk/global/GlobalAPI.kt | 6 + .../global/controller/WebViewController.kt | 46 ++++++ .../suwayomi/tachidesk/global/impl/WebView.kt | 143 ++++++++++++++++++ server/src/main/resources/selenium.html | 90 +++++++++++ 5 files changed, 287 insertions(+) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/global/controller/WebViewController.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt create mode 100644 server/src/main/resources/selenium.html diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 51e7d8d3c..5a21df51a 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -75,6 +75,8 @@ dependencies { implementation(libs.cron4j) implementation(libs.cronUtils) + + implementation("org.seleniumhq.selenium:selenium-java:4.11.0") } application { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/GlobalAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/GlobalAPI.kt index 174eb78ff..170250d5d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/global/GlobalAPI.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/GlobalAPI.kt @@ -10,8 +10,10 @@ package suwayomi.tachidesk.global import io.javalin.apibuilder.ApiBuilder.get import io.javalin.apibuilder.ApiBuilder.patch import io.javalin.apibuilder.ApiBuilder.path +import io.javalin.apibuilder.ApiBuilder.ws import suwayomi.tachidesk.global.controller.GlobalMetaController import suwayomi.tachidesk.global.controller.SettingsController +import suwayomi.tachidesk.global.controller.WebViewController object GlobalAPI { fun defineEndpoints() { @@ -23,5 +25,9 @@ object GlobalAPI { get("about", SettingsController.about) get("check-update", SettingsController.checkUpdate) } + path("webview") { + get("", WebViewController.webview) + ws("", WebViewController::webviewWS) + } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/controller/WebViewController.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/controller/WebViewController.kt new file mode 100644 index 000000000..fc069ad7f --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/controller/WebViewController.kt @@ -0,0 +1,46 @@ +package suwayomi.tachidesk.global.controller + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import io.javalin.http.ContentType +import io.javalin.http.HttpCode +import io.javalin.websocket.WsConfig +import suwayomi.tachidesk.global.impl.WebView +import suwayomi.tachidesk.server.util.handler +import suwayomi.tachidesk.server.util.withOperation + +object WebViewController { + /** returns some static info about the current app build */ + val webview = handler( + documentWith = { + withOperation { + summary("WebView") + description("Opens and browses WebView") + } + }, + behaviorOf = { ctx -> + ctx.contentType(ContentType.TEXT_HTML) + ctx.result(javaClass.getResourceAsStream("/selenium.html")!!) + }, + withResults = { + mime(HttpCode.OK, "text/html") + } + ) + + fun webviewWS(ws: WsConfig) { + ws.onConnect { ctx -> + WebView.addClient(ctx) + } + ws.onMessage { ctx -> + WebView.handleRequest(ctx) + } + ws.onClose { ctx -> + WebView.removeClient(ctx) + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt new file mode 100644 index 000000000..c9c15fa43 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt @@ -0,0 +1,143 @@ +package suwayomi.tachidesk.global.impl + +import io.javalin.websocket.WsContext +import io.javalin.websocket.WsMessageContext +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.double +import kotlinx.serialization.json.jsonPrimitive +import org.eclipse.jetty.websocket.api.CloseStatus +import org.openqa.selenium.By +import org.openqa.selenium.Dimension +import org.openqa.selenium.OutputType +import org.openqa.selenium.TakesScreenshot +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.interactions.Actions +import suwayomi.tachidesk.manga.impl.update.Websocket +import uy.kohesive.injekt.injectLazy +import java.io.Closeable +import java.util.concurrent.Executors + +@Serializable +sealed class WebViewEvent { + @SerialName("click") + data class Click( + val x: Int, + val y: Int + ) : WebViewEvent() + + @SerialName("keypress") + data class KeyPress( + val key: String + ) : WebViewEvent() +} + +object WebView : Websocket() { + val json: Json by injectLazy() + + var driver: SeleniumScreenshotServer? = null + + override fun addClient(ctx: WsContext) { + if (clients.isNotEmpty()) { + clients.forEach { + it.value.closeSession(CloseStatus(1001, "Other client connected")) + } + clients.clear() + } else { + driver = SeleniumScreenshotServer() + } + super.addClient(ctx) + } + + override fun removeClient(ctx: WsContext) { + super.removeClient(ctx) + if (clients.isEmpty()) { + driver?.close() + driver = null + } + } + + override fun notifyClient(ctx: WsContext, value: String?) { + if (value != null) { + ctx.send(value) + } + } + + override fun handleRequest(ctx: WsMessageContext) { + try { + val event = json.decodeFromString(ctx.message()) + driver?.handleEvent(event) + } catch (e: Exception) { + e.printStackTrace() + } + } +} + +class SeleniumScreenshotServer : Closeable { + companion object { + const val width = 1200 + const val height = 800 + } + + private val driver: WebDriver = ChromeDriver() + init { + driver.manage().window().size = Dimension(width, height) + driver.get("https://google.com") + } + private val actions = Actions(driver) + private val executor = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + + init { + GlobalScope.launch(executor) { + try { + while (isActive) { + // Capture screenshot + val screenshot = + (driver as TakesScreenshot).getScreenshotAs(OutputType.BASE64) + + // Send image data over the socket + WebView.notifyAllClients(screenshot) + delay(1000) // Adjust interval as needed + } + } catch (e: Exception) { + } + } + } + + fun handleEvent(jsonObject: JsonObject) { + when (jsonObject["type"]!!.jsonPrimitive.content) { + "click" -> { + val element = driver.findElement(By.tagName("body")) + val x = jsonObject["x"]!!.jsonPrimitive.double.toInt() + val y = jsonObject["y"]!!.jsonPrimitive.double.toInt() + if (x in 0..width && y in 0..height) { + actions.moveToLocation(x, y).click().perform() + } + } + "keypress" -> { + val key = jsonObject["key"]!!.jsonPrimitive.content + actions.keyDown(key).keyUp(key) + } + "back" -> { + driver.navigate().back() + } + "forward" -> { + driver.navigate().forward() + } + } + } + + override fun close() { + driver.close() + executor.cancel() + } +} diff --git a/server/src/main/resources/selenium.html b/server/src/main/resources/selenium.html new file mode 100644 index 000000000..a450c20d1 --- /dev/null +++ b/server/src/main/resources/selenium.html @@ -0,0 +1,90 @@ + + + + WebSocket Stream with Click Events + + + +
+ + +
+
+ + + + From ae55d313ca76861ecc9a6669090c5aede54f95ca Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sun, 6 Aug 2023 00:20:06 -0400 Subject: [PATCH 2/9] Get basic key input working --- .../kotlin/suwayomi/tachidesk/global/impl/WebView.kt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt index c9c15fa43..384b63892 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt @@ -17,6 +17,7 @@ import kotlinx.serialization.json.jsonPrimitive import org.eclipse.jetty.websocket.api.CloseStatus import org.openqa.selenium.By import org.openqa.selenium.Dimension +import org.openqa.selenium.Keys import org.openqa.selenium.OutputType import org.openqa.selenium.TakesScreenshot import org.openqa.selenium.WebDriver @@ -93,7 +94,6 @@ class SeleniumScreenshotServer : Closeable { driver.manage().window().size = Dimension(width, height) driver.get("https://google.com") } - private val actions = Actions(driver) private val executor = Executors.newSingleThreadExecutor().asCoroutineDispatcher() init { @@ -120,12 +120,18 @@ class SeleniumScreenshotServer : Closeable { val x = jsonObject["x"]!!.jsonPrimitive.double.toInt() val y = jsonObject["y"]!!.jsonPrimitive.double.toInt() if (x in 0..width && y in 0..height) { - actions.moveToLocation(x, y).click().perform() + Actions(driver).moveToLocation(x, y).click().perform() } } "keypress" -> { val key = jsonObject["key"]!!.jsonPrimitive.content - actions.keyDown(key).keyUp(key) + val keys: CharSequence = when (key) { + "Backspace" -> Keys.BACK_SPACE + "Tab" -> Keys.TAB + "Enter" -> Keys.ENTER + else -> key + } + Actions(driver).keyDown(keys).keyUp(keys).perform() } "back" -> { driver.navigate().back() From ee8fe560b478d03521889fbafee9dd8e77b7ac7d Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sun, 6 Aug 2023 00:27:53 -0400 Subject: [PATCH 3/9] Make webview headless --- .../suwayomi/tachidesk/global/impl/WebView.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt index 384b63892..539dbc97b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt @@ -22,6 +22,7 @@ import org.openqa.selenium.OutputType import org.openqa.selenium.TakesScreenshot import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.interactions.Actions import suwayomi.tachidesk.manga.impl.update.Websocket import uy.kohesive.injekt.injectLazy @@ -89,7 +90,11 @@ class SeleniumScreenshotServer : Closeable { const val height = 800 } - private val driver: WebDriver = ChromeDriver() + private val driver: WebDriver = run { + val options = ChromeOptions() + options.addArguments("--headless") + ChromeDriver(options) + } init { driver.manage().window().size = Dimension(width, height) driver.get("https://google.com") @@ -98,17 +103,17 @@ class SeleniumScreenshotServer : Closeable { init { GlobalScope.launch(executor) { - try { - while (isActive) { + while (isActive) { + try { // Capture screenshot val screenshot = (driver as TakesScreenshot).getScreenshotAs(OutputType.BASE64) - // Send image data over the socket WebView.notifyAllClients(screenshot) delay(1000) // Adjust interval as needed + } catch (e: Exception) { + e.printStackTrace() } - } catch (e: Exception) { } } } From e0085f1d76ee42360f6e6375323fd0c57373fb2d Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sun, 6 Aug 2023 11:24:17 -0400 Subject: [PATCH 4/9] Open source, manga, or chapter in webview --- .../suwayomi/tachidesk/global/impl/WebView.kt | 41 +++++++++++-------- server/src/main/resources/selenium.html | 14 +++++++ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt index 539dbc97b..4de0dcb13 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/WebView.kt @@ -1,5 +1,6 @@ package suwayomi.tachidesk.global.impl +import eu.kanade.tachiyomi.source.online.HttpSource import io.javalin.websocket.WsContext import io.javalin.websocket.WsMessageContext import kotlinx.coroutines.GlobalScope @@ -8,14 +9,15 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.double +import kotlinx.serialization.json.intOrNull import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.longOrNull import org.eclipse.jetty.websocket.api.CloseStatus -import org.openqa.selenium.By +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.transactions.transaction import org.openqa.selenium.Dimension import org.openqa.selenium.Keys import org.openqa.selenium.OutputType @@ -25,24 +27,13 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.interactions.Actions import suwayomi.tachidesk.manga.impl.update.Websocket +import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaTable import uy.kohesive.injekt.injectLazy import java.io.Closeable import java.util.concurrent.Executors -@Serializable -sealed class WebViewEvent { - @SerialName("click") - data class Click( - val x: Int, - val y: Int - ) : WebViewEvent() - - @SerialName("keypress") - data class KeyPress( - val key: String - ) : WebViewEvent() -} - object WebView : Websocket() { val json: Json by injectLazy() @@ -121,7 +112,6 @@ class SeleniumScreenshotServer : Closeable { fun handleEvent(jsonObject: JsonObject) { when (jsonObject["type"]!!.jsonPrimitive.content) { "click" -> { - val element = driver.findElement(By.tagName("body")) val x = jsonObject["x"]!!.jsonPrimitive.double.toInt() val y = jsonObject["y"]!!.jsonPrimitive.double.toInt() if (x in 0..width && y in 0..height) { @@ -144,6 +134,21 @@ class SeleniumScreenshotServer : Closeable { "forward" -> { driver.navigate().forward() } + "loadsource" -> { + val sourceId = jsonObject["source"]!!.jsonPrimitive.longOrNull ?: return + val source = getCatalogueSourceOrNull(sourceId) as? HttpSource ?: return + driver.get(source.baseUrl) + } + "loadmanga" -> { + val mangaId = jsonObject["manga"]!!.jsonPrimitive.intOrNull ?: return + val manga = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull() } ?: return + driver.get(manga[MangaTable.realUrl] ?: return) + } + "loadchapter" -> { + val chapterId = jsonObject["chapter"]!!.jsonPrimitive.intOrNull ?: return + val chapter = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.firstOrNull() } ?: return + driver.get(chapter[ChapterTable.realUrl] ?: return) + } } } diff --git a/server/src/main/resources/selenium.html b/server/src/main/resources/selenium.html index a450c20d1..3b40fc7fb 100644 --- a/server/src/main/resources/selenium.html +++ b/server/src/main/resources/selenium.html @@ -33,7 +33,21 @@ const forwardButton = document.getElementById('forwardButton'); const socket = new WebSocket(window.location.href.replace(/^http/,'ws')); + // Read the 'source' query parameter from the URL + const queryParams = new URLSearchParams(window.location.search); + const source = queryParams.get('source'); + const manga = queryParams.get('manga'); + const chapter = queryParams.get('chapter'); + socket.onopen = () => { + if (source !== undefined && source !== null) { + socket.send(JSON.stringify({ type: 'loadsource', source: source })); + } else if (manga !== undefined && manga !== null) { + socket.send(JSON.stringify({ type: 'loadmanga', manga: manga })); + } else if (chapter !== undefined && chapter !== null) { + socket.send(JSON.stringify({ type: 'loadchapter', chapter: chapter })); + } + console.log('WebSocket connection opened'); }; From 27e047e8572da70ac1520d90df4f05e47697b872 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sun, 6 Aug 2023 12:22:39 -0400 Subject: [PATCH 5/9] Put back and forward on top --- server/src/main/resources/selenium.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/resources/selenium.html b/server/src/main/resources/selenium.html index 3b40fc7fb..830af752c 100644 --- a/server/src/main/resources/selenium.html +++ b/server/src/main/resources/selenium.html @@ -5,8 +5,9 @@