trying to bundle the changes

buildSrc/src/main/kotlin/Constants.kt

@@ -12,7 +12,7 @@ const val MainClass = "suwayomi.tachidesk.MainKt"
 // should be bumped with each stable release
 val tachideskVersion = System.getenv("ProductVersion") ?: "v0.7.0"
 
-val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r983"
+val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r1045"
 
 // counts commits on the master branch
 val tachideskRevision = runCatching {

gradle/libs.versions.toml

@@ -10,7 +10,6 @@ dex2jar = "v60"
 rhino = "1.7.14"
 settings = "1.0.0-RC"
 twelvemonkeys = "3.9.4"
-playwright = "1.28.0"
 
 [libraries]
 # Kotlin
@@ -95,9 +94,6 @@ appdirs = "net.harawata:appdirs:1.2.1"
 zip4j = "net.lingala.zip4j:zip4j:2.11.2"
 junrar = "com.github.junrar:junrar:7.5.3"
 
-# CloudflareInterceptor
-playwright = { module = "com.microsoft.playwright:playwright", version.ref = "playwright" }
-
 # AES/CBC/PKCS7Padding Cypher provider
 bouncycastle = "org.bouncycastle:bcprov-jdk18on:1.72"
 

scripts/bundler.sh

@@ -26,8 +26,6 @@ main() {
   set -- "${POSITIONAL_ARGS[@]}"
 
   OS="$1"
-  PLAYWRIGHT_VERSION="$(cat gradle/libs.versions.toml | grep -oP "playwright = \"\K([0-9\.]*)(?=\")")"
-  PLAYWRIGHT_REVISION="$(curl --silent "https://raw.githubusercontent.com/microsoft/playwright/v$PLAYWRIGHT_VERSION/packages/playwright-core/browsers.json" 2>&1 | grep -ozP "\"name\": \"chromium\",\n *\"revision\": \"\K[0-9]*")"
   JAR="$(ls server/build/*.jar | tail -n1)"
   RELEASE_NAME="$(echo "${JAR%.*}" | xargs basename)-$OS"
   RELEASE_VERSION="$(tmp="${JAR%-*}"; echo "${tmp##*-}" | tr -d v)"
@@ -127,8 +125,9 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
-      PLAYWRIGHT_PLATFORM="win64"
-      setup_playwright
+      PYTHON=Winpython64-3.10.9.0dot.exe
+      PYTHON_URL=https://github.com/winpython/winpython/releases/download/5.3.20221233/Winpython64-3.10.9.0dot.exe
+      setup_undetected_chromedriver_and_python
 
       RELEASE="$RELEASE_NAME.zip"
       make_windows_bundle
@@ -285,9 +284,23 @@ make_windows_package() {
     "$RELEASE_NAME/jre.wxs" "$RELEASE_NAME/electron.wxs" -o "$RELEASE"
 }
 
-setup_playwright() {
-  mkdir "$RELEASE_NAME/bin"
-  curl -L "https://playwright.azureedge.net/builds/chromium/$PLAYWRIGHT_REVISION/chromium-$PLAYWRIGHT_PLATFORM.zip" -o "$RELEASE_NAME/bin/chromium.zip"
+setup_python() {
+  mkdir "$RELEASE_NAME/"
+  curl -L "$PYTHON_URL" -o "$PYTHON"
+  7z x $PYTHON
+  mv WPy64-31090/python-3.10.9.amd64 "$RELEASE_NAME/python"
+}
+
+setup_undetected_chromedriver() {
+  curl -L "https://github.com/Suwayomi/undetected-chromedriver/archive/refs/heads/master.zip" -o undetected-chromedriver-master.zip
+  unzip undetected-chromedriver-master.zip
+  mv undetected-chromedriver-master "$RELEASE_NAME/undetected-chromedriver"
+}
+
+
+setup_undetected_chromedriver_and_python() {
+  setup_python
+  setup_undetected_chromedriver
 }
 
 # Error handler

server/build.gradle.kts

@@ -50,9 +50,6 @@ dependencies {
     implementation(libs.zip4j)
     implementation(libs.junrar)
 
-    // CloudflareInterceptor
-    implementation(libs.playwright)
-
     // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga
     implementation(libs.bouncycastle)
 

server/src/main/java/suwayomi/tachidesk/server/util/DriverJar.java

@@ -1,212 +0,0 @@
-/*
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package suwayomi.tachidesk.server.util;
-
-import com.microsoft.playwright.impl.driver.Driver;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.*;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Copy of <a href="https://github.com/microsoft/playwright-java/blob/8c0231b0f739656e8a86bc58fca9ee778ddc571b/driver-bundle/src/main/java/com/microsoft/playwright/impl/driver/jar/DriverJar.java">DriverJar</a>
- * with support for pre-installing chromium and only supports chromium playwright
- */
-public class DriverJar extends Driver {
-  private static final String PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD";
-  private static final String SELENIUM_REMOTE_URL = "SELENIUM_REMOTE_URL";
-  static final String PLAYWRIGHT_NODEJS_PATH = "PLAYWRIGHT_NODEJS_PATH";
-  private final Path driverTempDir;
-  private Path preinstalledNodePath;
-
-  public DriverJar() throws IOException {
-    // Allow specifying custom path for the driver installation
-    // See https://github.com/microsoft/playwright-java/issues/728
-    String alternativeTmpdir = System.getProperty("playwright.driver.tmpdir");
-    String prefix = "playwright-java-";
-    driverTempDir = alternativeTmpdir == null
-            ? Files.createTempDirectory(prefix)
-            : Files.createTempDirectory(Paths.get(alternativeTmpdir), prefix);
-    driverTempDir.toFile().deleteOnExit();
-    String nodePath = System.getProperty("playwright.nodejs.path");
-    if (nodePath != null) {
-      preinstalledNodePath = Paths.get(nodePath);
-      if (!Files.exists(preinstalledNodePath)) {
-        throw new RuntimeException("Invalid Node.js path specified: " + nodePath);
-      }
-    }
-    logMessage("created DriverJar: " + driverTempDir);
-  }
-
-  @Override
-  protected void initialize(Boolean installBrowsers) throws Exception {
-    if (preinstalledNodePath == null && env.containsKey(PLAYWRIGHT_NODEJS_PATH)) {
-      preinstalledNodePath = Paths.get(env.get(PLAYWRIGHT_NODEJS_PATH));
-      if (!Files.exists(preinstalledNodePath)) {
-        throw new RuntimeException("Invalid Node.js path specified: " + preinstalledNodePath);
-      }
-    } else if (preinstalledNodePath != null) {
-      // Pass the env variable to the driver process.
-      env.put(PLAYWRIGHT_NODEJS_PATH, preinstalledNodePath.toString());
-    }
-    extractDriverToTempDir();
-    logMessage("extracted driver from jar to " + driverPath());
-    if (installBrowsers)
-      installBrowsers(env);
-  }
-
-  private void installBrowsers(Map<String, String> env) throws IOException, InterruptedException {
-    String skip = env.get(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD);
-    if (skip == null) {
-      skip = System.getenv(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD);
-    }
-    if (skip != null && !"0".equals(skip) && !"false".equals(skip)) {
-      System.out.println("Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set");
-      return;
-    }
-    if (env.get(SELENIUM_REMOTE_URL) != null || System.getenv(SELENIUM_REMOTE_URL) != null) {
-      logMessage("Skipping browsers download because `SELENIUM_REMOTE_URL` env variable is set");
-      return;
-    }
-    Chromium.preinstall(platformDir());
-    Path driver = driverPath();
-    if (!Files.exists(driver)) {
-      throw new RuntimeException("Failed to find driver: " + driver);
-    }
-    ProcessBuilder pb = createProcessBuilder();
-    pb.command().add("install");
-    pb.command().add("chromium");
-    pb.redirectError(ProcessBuilder.Redirect.INHERIT);
-    pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
-    Process p = pb.start();
-    boolean result = p.waitFor(10, TimeUnit.MINUTES);
-    if (!result) {
-      p.destroy();
-      throw new RuntimeException("Timed out waiting for browsers to install");
-    }
-    if (p.exitValue() != 0) {
-      throw new RuntimeException("Failed to install browsers, exit code: " + p.exitValue());
-    }
-  }
-
-  private static boolean isExecutable(Path filePath) {
-    String name = filePath.getFileName().toString();
-    return name.endsWith(".sh") || name.endsWith(".exe") || !name.contains(".");
-  }
-
-  private FileSystem initFileSystem(URI uri) throws IOException {
-    try {
-      return FileSystems.newFileSystem(uri, Collections.emptyMap());
-    } catch (FileSystemAlreadyExistsException e) {
-      return null;
-    }
-  }
-
-  public static URI getDriverResourceURI() throws URISyntaxException {
-    ClassLoader classloader = Thread.currentThread().getContextClassLoader();
-    return classloader.getResource("driver/" + platformDir()).toURI();
-  }
-
-  void extractDriverToTempDir() throws URISyntaxException, IOException {
-    URI originalUri = getDriverResourceURI();
-    URI uri = maybeExtractNestedJar(originalUri);
-
-    // Create zip filesystem if loading from jar.
-    try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? initFileSystem(uri) : null) {
-      Path srcRoot = Paths.get(uri);
-      // jar file system's .relativize gives wrong results when used with
-      // spring-boot-maven-plugin, convert to the default filesystem to
-      // have predictable results.
-      // See https://github.com/microsoft/playwright-java/issues/306
-      Path srcRootDefaultFs = Paths.get(srcRoot.toString());
-      Files.walk(srcRoot).forEach(fromPath -> {
-        if (preinstalledNodePath != null) {
-          String fileName = fromPath.getFileName().toString();
-          if ("node.exe".equals(fileName) || "node".equals(fileName)) {
-            return;
-          }
-        }
-        Path relative = srcRootDefaultFs.relativize(Paths.get(fromPath.toString()));
-        Path toPath = driverTempDir.resolve(relative.toString());
-        try {
-          if (Files.isDirectory(fromPath)) {
-            Files.createDirectories(toPath);
-          } else {
-            Files.copy(fromPath, toPath);
-            if (isExecutable(toPath)) {
-              toPath.toFile().setExecutable(true, true);
-            }
-          }
-          toPath.toFile().deleteOnExit();
-        } catch (IOException e) {
-          throw new RuntimeException("Failed to extract driver from " + uri + ", full uri: " + originalUri, e);
-        }
-      });
-    }
-  }
-
-  private URI maybeExtractNestedJar(final URI uri) throws URISyntaxException {
-    if (!"jar".equals(uri.getScheme())) {
-      return uri;
-    }
-    final String JAR_URL_SEPARATOR = "!/";
-    String[] parts = uri.toString().split("!/");
-    if (parts.length != 3) {
-      return uri;
-    }
-    String innerJar = String.join(JAR_URL_SEPARATOR, parts[0], parts[1]);
-    URI jarUri = new URI(innerJar);
-    try (FileSystem fs = FileSystems.newFileSystem(jarUri, Collections.emptyMap())) {
-      Path fromPath = Paths.get(jarUri);
-      Path toPath = driverTempDir.resolve(fromPath.getFileName().toString());
-      Files.copy(fromPath, toPath);
-      toPath.toFile().deleteOnExit();
-      return new URI("jar:" + toPath.toUri() + JAR_URL_SEPARATOR + parts[2]);
-    } catch (IOException e) {
-      throw new RuntimeException("Failed to extract driver's nested .jar from " + jarUri + "; full uri: " + uri, e);
-    }
-  }
-
-  private static String platformDir() {
-    String name = System.getProperty("os.name").toLowerCase();
-    String arch = System.getProperty("os.arch").toLowerCase();
-
-    if (name.contains("windows")) {
-      return "win32_x64";
-    }
-    if (name.contains("linux")) {
-      if (arch.equals("aarch64")) {
-        return "linux-arm64";
-      } else {
-        return "linux";
-      }
-    }
-    if (name.contains("mac os x")) {
-      return "mac";
-    }
-    throw new RuntimeException("Unexpected os.name value: " + name);
-  }
-
-  @Override
-  protected Path driverDir() {
-    return driverTempDir;
-  }
-}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -1,24 +1,29 @@
 package eu.kanade.tachiyomi.network.interceptor
 
-import com.microsoft.playwright.Browser
-import com.microsoft.playwright.BrowserType.LaunchOptions
-import com.microsoft.playwright.Page
-import com.microsoft.playwright.Playwright
-import com.microsoft.playwright.PlaywrightException
 import eu.kanade.tachiyomi.network.NetworkHelper
-import eu.kanade.tachiyomi.network.interceptor.CFClearance.resolveWithWebView
+import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.resolveWithWebView
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import mu.KotlinLogging
 import okhttp3.Cookie
 import okhttp3.HttpUrl
 import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
-import suwayomi.tachidesk.server.ServerConfig
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
 import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
+import java.io.BufferedReader
+import java.io.Closeable
+import java.io.File
 import java.io.IOException
-import kotlin.time.Duration.Companion.seconds
-import kotlin.time.DurationUnit
+import java.io.InputStreamReader
+import java.io.PrintWriter
+import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
 class CloudflareInterceptor : Interceptor {
     private val logger = KotlinLogging.logger {}
@@ -38,10 +43,12 @@ class CloudflareInterceptor : Interceptor {
             return originalResponse
         }
 
-        throw IOException("playwrite is diabled for v0.6.7")
-
         logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
 
+        if (!serverConfig.webviewEnabled) {
+            throw CloudflareBypassException("Webview is disabled, enable it in server config")
+        }
+
         return try {
             originalResponse.close()
             network.cookies.remove(originalRequest.url.toUri())
@@ -63,43 +70,43 @@ class CloudflareInterceptor : Interceptor {
     }
 }
 
-/*
- * This class is ported from https://github.com/vvanglro/cf-clearance
- * The original code is licensed under Apache 2.0
-*/
-object CFClearance {
+object CloudflareBypasser {
     private val logger = KotlinLogging.logger {}
     private val network: NetworkHelper by injectLazy()
 
-    init {
-        // Fix the default DriverJar issue by providing our own implementation
-        // ref: https://github.com/microsoft/playwright-java/issues/1138
-        System.setProperty("playwright.driver.impl", "suwayomi.tachidesk.server.util.DriverJar")
-    }
-
     fun resolveWithWebView(originalRequest: Request): Request {
         val url = originalRequest.url.toString()
 
         logger.debug { "resolveWithWebView($url)" }
 
-        val cookies = Playwright.create().use { playwright ->
-            playwright.chromium().launch(
-                LaunchOptions()
-                    .setHeadless(false)
-                    .apply {
-                        if (serverConfig.socksProxyEnabled) {
-                            setProxy("socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}")
-                        }
-                    }
-            ).use { browser ->
-                val userAgent = originalRequest.header("User-Agent")
-                if (userAgent != null) {
-                    browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
-                        browserContext.newPage().use { getCookies(it, url) }
-                    }
-                } else {
-                    browser.newPage().use { getCookies(it, url) }
+        val cookies = PythonInterpreter.create().use { py ->
+            try {
+                py.exec("import undetected_chromedriver as uc")
+
+                py.exec("options = uc.ChromeOptions()")
+
+                if (serverConfig.socksProxyEnabled) {
+                    val proxy = "socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}"
+                    py.exec("options.add_argument('--proxy-server=$proxy')")
                 }
+//                py.exec("driver = uc.Chrome(options=options)")
+                py.exec("driver = uc.Chrome(options=options, driver_executable_path='${py.chromedriverPath.replace("\\","\\\\")}', version_main=111)")
+
+                // TODO: handle custom userAgent
+//                val userAgent = originalRequest.header("User-Agent")
+
+//                if (userAgent != null) {
+//                    browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
+//                        browserContext.newPage().use { getCookies(it, url) }
+//                    }
+//                } else {
+//                    browser.newPage().use { getCookies(it, url) }
+//                }
+                py.exec("driver.get('$url')")
+
+                getCookies(py)
+            } finally {
+                py.exec("driver.quit()")
             }
         }
 
@@ -139,44 +146,64 @@ object CFClearance {
 
     fun getWebViewUserAgent(): String {
         return try {
-            throw PlaywrightException("playwrite is diabled for v0.6.7")
-
-            Playwright.create().use { playwright ->
-                playwright.chromium().launch(
-                    LaunchOptions()
-                        .setHeadless(true)
-                ).use { browser ->
-                    browser.newPage().use { page ->
-                        val userAgent = page.evaluate("() => {return navigator.userAgent}") as String
-                        logger.debug { "WebView User-Agent is $userAgent" }
-                        return userAgent
-                    }
-                }
+            if (!serverConfig.webviewEnabled) {
+                throw CloudflareBypassException("Webview is disabled, enable it in server config")
             }
-        } catch (e: PlaywrightException) {
-            // Playwright might fail on headless environments like docker
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
+
+            PythonInterpreter.create().use { py ->
+                py.exec("import undetected_chromedriver as uc")
+
+                py.exec("options = uc.ChromeOptions()")
+                py.exec("options.add_argument('--headless')")
+                py.exec("options.add_argument('--disable-gpu')")
+//                py.exec("driver = uc.Chrome(options=options)")
+                py.exec("driver = uc.Chrome(options=options, driver_executable_path='${py.chromedriverPath.replace("\\","\\\\")}', version_main=111)")
+
+                py.exec("userAgent = driver.execute_script('return navigator.userAgent')")
+                val userAgent = py.getValue("userAgent")
+                py.exec("driver.quit()")
+
+                userAgent
+            }
+        } catch (e: Exception) {
+            // Webview might fail on headless environments like docker
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
         }
     }
 
-    private fun getCookies(page: Page, url: String): List<Cookie> {
-        applyStealthInitScripts(page)
-        page.navigate(url)
-        val challengeResolved = waitForChallengeResolve(page)
+    @Serializable
+    data class PythonSeleniumCookie(
+        val domain: String,
+        val expiry: Long?,
+        val httpOnly: Boolean,
+        val name: String,
+        val path: String,
+        val sameSite: String,
+        val secure: Boolean,
+        val value: String
+    )
+
+    private val json by DI.global.instance<Json>()
+    private fun getCookies(py: PythonInterpreter): List<Cookie> {
+        val challengeResolved = waitForChallengeResolve(py)
 
         return if (challengeResolved) {
-            val cookies = page.context().cookies()
+            py.exec("import json")
+            py.exec("cookies = json.dumps(driver.get_cookies())")
+            val cookiesJson = py.getValue("cookies")
+            val cookies = json.decodeFromString<List<PythonSeleniumCookie>>(cookiesJson)
 
             logger.debug {
-                val userAgent = page.evaluate("() => {return navigator.userAgent}")
-                "Playwright User-Agent is $userAgent"
+                py.exec("userAgent = driver.execute_script('return navigator.userAgent')")
+                val userAgent = py.getValue("userAgent")
+                "Webview User-Agent is $userAgent"
             }
 
-            // Convert PlayWright cookies to OkHttp cookies
+            // Convert Webview cookies to OkHttp cookies
             cookies.map {
                 Cookie.Builder()
                     .domain(it.domain.removePrefix("."))
-                    .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE)
+                    .expiresAt(it.expiry?.times(1000) ?: Long.MAX_VALUE)
                     .name(it.name)
                     .path(it.path)
                     .value(it.value).apply {
@@ -185,39 +212,18 @@ object CFClearance {
                     }.build()
             }
         } else {
-            logger.debug { "Cloudflare challenge failed to resolve" }
-            throw CloudflareBypassException()
+            throw CloudflareBypassException("Cloudflare challenge failed to resolve")
         }
     }
 
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18
-    private val stealthInitScripts by lazy {
-        arrayOf(
-            ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText()
-        )
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76
-    private fun applyStealthInitScripts(page: Page) {
-        for (script in stealthInitScripts) {
-            page.addInitScript(script)
-        }
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21
-    private fun waitForChallengeResolve(page: Page): Boolean {
-        // sometimes the user has to solve the captcha challenge manually, potentially wait a long time
+    private fun waitForChallengeResolve(py: PythonInterpreter): Boolean {
+        // sometimes the user has to solve the captcha challenge manually and multiple times, potentially wait a long time
         val timeoutSeconds = 120
         repeat(timeoutSeconds) {
-            page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
+            TimeUnit.SECONDS.sleep(1)
             val success = try {
-                page.querySelector("#challenge-form") == null
+                py.exec("r = driver.execute_script('return document.querySelector(\"#challenge-form\") == null')")
+                py.getValue("r").lowercase().toBoolean()
             } catch (e: Exception) {
                 logger.debug(e) { "query Error" }
                 false
@@ -226,6 +232,113 @@ object CFClearance {
         }
         return false
     }
-
-    private class CloudflareBypassException : Exception()
+}
+private class CloudflareBypassException(message: String?) : Exception(message)
+
+class PythonInterpreter
+private constructor(private val process: Process, val chromedriverPath: String) : Closeable {
+    private val stdin = process.outputStream
+    private val stdout = process.inputStream
+    private val stderr = process.errorStream
+
+    private val stdinWriter = PrintWriter(stdin)
+    private val stdoutReader = BufferedReader(InputStreamReader(stdout))
+    private val stderrReader = BufferedReader(InputStreamReader(stderr))
+
+    private fun rawExec(command: String) {
+        stdinWriter.println(command)
+        stdinWriter.flush()
+    }
+
+    val BUFF_SIZE = 102400
+    fun exec(command: String) {
+        logger.debug { "Python Command: $command" }
+        rawExec(command)
+        makeSureExecDone()
+    }
+
+    private val commandOutputs = mutableListOf<String>()
+
+    fun makeSureExecDone() {
+        val makeSureString = "PYTHON_IS_READY"
+
+        rawExec("print('$makeSureString')")
+        var line: String?
+        do {
+            line = stdoutReader.readLine()
+            if (line != makeSureString) {
+                commandOutputs.add(line)
+            }
+        } while (line != makeSureString)
+
+        val pyError = buildString {
+            while (stderrReader.ready())
+                append(stderr.read().toChar())
+        }
+        if (pyError.isNotEmpty()) {
+            println("Python STDERR: $pyError")
+        }
+    }
+
+    fun getValue(variableName: String): String {
+        exec("print($variableName)")
+        return commandOutputs.last()
+    }
+
+    private fun flushStdoutReader() {
+        var line: String?
+        while (stdoutReader.ready()) {
+            val line = stdoutReader.readLine()
+        }
+    }
+
+    fun destroy() {
+        stdinWriter.close()
+        stdoutReader.close()
+        stderr.close()
+        process.destroy()
+    }
+    override fun close() {
+        destroy()
+    }
+    companion object {
+        private val logger = KotlinLogging.logger {}
+
+        fun create(pythonPath: String, workingDir: String, pythonStartupFile: String, chromedriverPath: String): PythonInterpreter {
+            val processBuilder = ProcessBuilder()
+                .command(pythonPath, "-i", "-q")
+            processBuilder.directory(File(workingDir))
+
+            val environment = processBuilder.environment()
+            environment["PYTHONSTARTUP"] = pythonStartupFile
+
+            val process = processBuilder.start()
+
+            return PythonInterpreter(process, chromedriverPath)
+        }
+
+        fun create(): PythonInterpreter {
+            val uc = Paths.get(serverConfig.undetectedChromePath).toAbsolutePath().toString()
+            logger.debug { "absolute path to undetected-chromedriver: $uc" }
+
+            val (pythonPath, chromedriverPath) = if (System.getProperty("os.name").startsWith("Windows")) {
+                arrayOf(
+                    "$uc\\venv\\Scripts\\python.exe",
+                    "$uc\\chromedriver.exe"
+                )
+            } else {
+                arrayOf(
+                    "$uc/venv/bin/python",
+                    "$uc/chromedriver"
+                )
+            }
+
+            return create(
+                pythonPath,
+                uc,
+                "$uc/console.py",
+                chromedriverPath
+            )
+        }
+    }
 }

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -3,7 +3,7 @@ package eu.kanade.tachiyomi.source.online
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.asObservableSuccess
-import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent
+import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.getWebViewUserAgent
 import eu.kanade.tachiyomi.network.newCallWithProgress
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt

@@ -1,5 +1,12 @@
 package suwayomi.tachidesk.manga.controller
 
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import io.javalin.http.HttpCode
 import io.javalin.websocket.WsConfig
@@ -20,13 +27,6 @@ import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
 import suwayomi.tachidesk.server.util.withOperation
 
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
 object UpdateController {
     private val logger = KotlinLogging.logger { }
 
@@ -115,7 +115,7 @@ object UpdateController {
 
         updater.addMangasToQueue(
             mangasToUpdate
-                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
+                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
         )
     }
 

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -45,6 +45,10 @@ class ServerConfig(config: Config, moduleName: String = MODULE_NAME) : SystemPro
     val debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config)
     val systemTrayEnabled: Boolean by overridableConfig
 
+    // Webview
+    val webviewEnabled: Boolean by overridableConfig
+    val undetectedChromePath: String by overridableConfig
+
     companion object {
         fun register(config: Config) = ServerConfig(config.getConfig(MODULE_NAME))
     }

server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt

@@ -7,7 +7,6 @@ package suwayomi.tachidesk.server.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import kotlinx.serialization.json.Json
 import mu.KotlinLogging
 import net.lingala.zip4j.ZipFile
 import org.kodein.di.DI
@@ -16,7 +15,6 @@ import org.kodein.di.instance
 import suwayomi.tachidesk.server.ApplicationDirs
 import suwayomi.tachidesk.server.BuildConfig
 import suwayomi.tachidesk.server.serverConfig
-import uy.kohesive.injekt.injectLazy
 import java.io.File
 import java.io.InputStream
 import java.net.HttpURLConnection
@@ -26,7 +24,6 @@ import java.security.MessageDigest
 
 private val logger = KotlinLogging.logger {}
 private val applicationDirs by DI.global.instance<ApplicationDirs>()
-private val json: Json by injectLazy()
 private val tmpDir = System.getProperty("java.io.tmpdir")
 
 private fun ByteArray.toHex(): String = joinToString(separator = "") { eachByte -> "%02x".format(eachByte) }

server/src/main/resources/cloudflare-js/canvas.fingerprinting.js

@@ -1,28 +0,0 @@
-(function () {
-    const ORIGINAL_CANVAS = HTMLCanvasElement.prototype[name];
-    Object.defineProperty(HTMLCanvasElement.prototype, name, {
-        "value": function () {
-            var shift = {
-                'r': Math.floor(Math.random() * 10) - 5,
-                'g': Math.floor(Math.random() * 10) - 5,
-                'b': Math.floor(Math.random() * 10) - 5,
-                'a': Math.floor(Math.random() * 10) - 5
-            };
-            var width = this.width,
-                height = this.height,
-                context = this.getContext("2d");
-            var imageData = context.getImageData(0, 0, width, height);
-            for (var i = 0; i < height; i++) {
-                for (var j = 0; j < width; j++) {
-                    var n = ((i * (width * 4)) + (j * 4));
-                    imageData.data[n + 0] = imageData.data[n + 0] + shift.r;
-                    imageData.data[n + 1] = imageData.data[n + 1] + shift.g;
-                    imageData.data[n + 2] = imageData.data[n + 2] + shift.b;
-                    imageData.data[n + 3] = imageData.data[n + 3] + shift.a;
-                }
-            }
-            context.putImageData(imageData, 0, 0);
-            return ORIGINAL_CANVAS.apply(this, arguments);
-        }
-    });
-})(this);

server/src/main/resources/cloudflare-js/chrome.global.js

@@ -1,52 +0,0 @@
-Object.defineProperty(window, 'chrome', {
-    value: new Proxy(window.chrome, {
-        has: (target, key) => true,
-        get: (target, key) => {
-            return {
-                app: {
-                    isInstalled: false,
-                },
-                webstore: {
-                    onInstallStageChanged: {},
-                    onDownloadProgress: {},
-                },
-                runtime: {
-                    PlatformOs: {
-                        MAC: 'mac',
-                        WIN: 'win',
-                        ANDROID: 'android',
-                        CROS: 'cros',
-                        LINUX: 'linux',
-                        OPENBSD: 'openbsd',
-                    },
-                    PlatformArch: {
-                        ARM: 'arm',
-                        X86_32: 'x86-32',
-                        X86_64: 'x86-64',
-                    },
-                    PlatformNaclArch: {
-                        ARM: 'arm',
-                        X86_32: 'x86-32',
-                        X86_64: 'x86-64',
-                    },
-                    RequestUpdateCheckStatus: {
-                        THROTTLED: 'throttled',
-                        NO_UPDATE: 'no_update',
-                        UPDATE_AVAILABLE: 'update_available',
-                    },
-                    OnInstalledReason: {
-                        INSTALL: 'install',
-                        UPDATE: 'update',
-                        CHROME_UPDATE: 'chrome_update',
-                        SHARED_MODULE_UPDATE: 'shared_module_update',
-                    },
-                    OnRestartRequiredReason: {
-                        APP_UPDATE: 'app_update',
-                        OS_UPDATE: 'os_update',
-                        PERIODIC: 'periodic',
-                    },
-                },
-            }
-        }
-    })
-});

server/src/main/resources/cloudflare-js/chrome.plugin.js

@@ -1,203 +0,0 @@
-(function () {
-    const plugin0 = Object.create(Plugin.prototype);
-
-    const mimeType0 = Object.create(MimeType.prototype);
-    const mimeType1 = Object.create(MimeType.prototype);
-    Object.defineProperties(mimeType0, {
-        type: {
-            get: () => 'application/pdf',
-        },
-        suffixes: {
-            get: () => 'pdf',
-        },
-    });
-
-    Object.defineProperties(mimeType1, {
-        type: {
-            get: () => 'text/pdf',
-        },
-        suffixes: {
-            get: () => 'pdf',
-        },
-    });
-
-    Object.defineProperties(plugin0, {
-        name: {
-            get: () => 'Chrome PDF Viewer',
-        },
-        description: {
-            get: () => 'Portable Document Format',
-        },
-        0: {
-            get: () => {
-                return mimeType0;
-            },
-        },
-        1: {
-            get: () => {
-                return mimeType1;
-            },
-        },
-        length: {
-            get: () => 2,
-        },
-        filename: {
-            get: () => 'internal-pdf-viewer',
-        },
-    });
-
-    const plugin1 = Object.create(Plugin.prototype);
-    Object.defineProperties(plugin1, {
-        name: {
-            get: () => 'Chromium PDF Viewer',
-        },
-        description: {
-            get: () => 'Portable Document Format',
-        },
-        0: {
-            get: () => {
-                return mimeType0;
-            },
-        },
-        1: {
-            get: () => {
-                return mimeType1;
-            },
-        },
-        length: {
-            get: () => 2,
-        },
-        filename: {
-            get: () => 'internal-pdf-viewer',
-        },
-    });
-
-    const plugin2 = Object.create(Plugin.prototype);
-    Object.defineProperties(plugin2, {
-        name: {
-            get: () => 'Microsoft Edge PDF Viewer',
-        },
-        description: {
-            get: () => 'Portable Document Format',
-        },
-        0: {
-            get: () => {
-                return mimeType0;
-            },
-        },
-        1: {
-            get: () => {
-                return mimeType1;
-            },
-        },
-        length: {
-            get: () => 2,
-        },
-        filename: {
-            get: () => 'internal-pdf-viewer',
-        },
-    });
-
-    const plugin3 = Object.create(Plugin.prototype);
-    Object.defineProperties(plugin3, {
-        name: {
-            get: () => 'PDF Viewer',
-        },
-        description: {
-            get: () => 'Portable Document Format',
-        },
-        0: {
-            get: () => {
-                return mimeType0;
-            },
-        },
-        1: {
-            get: () => {
-                return mimeType1;
-            },
-        },
-        length: {
-            get: () => 2,
-        },
-        filename: {
-            get: () => 'internal-pdf-viewer',
-        },
-    });
-
-    const plugin4 = Object.create(Plugin.prototype);
-    Object.defineProperties(plugin4, {
-        name: {
-            get: () => 'WebKit built-in PDF',
-        },
-        description: {
-            get: () => 'Portable Document Format',
-        },
-        0: {
-            get: () => {
-                return mimeType0;
-            },
-        },
-        1: {
-            get: () => {
-                return mimeType1;
-            },
-        },
-        length: {
-            get: () => 2,
-        },
-        filename: {
-            get: () => 'internal-pdf-viewer',
-        },
-    });
-
-    const pluginArray = Object.create(PluginArray.prototype);
-
-    pluginArray['0'] = plugin0;
-    pluginArray['1'] = plugin1;
-    pluginArray['2'] = plugin2;
-    pluginArray['3'] = plugin3;
-    pluginArray['4'] = plugin4;
-
-    let refreshValue;
-
-    Object.defineProperties(pluginArray, {
-        length: {
-            get: () => 5,
-        },
-        item: {
-            value: (index) => {
-                if (index > 4294967295) {
-                    index = index % 4294967296;
-                }
-                switch (index) {
-                    case 0:
-                        return plugin3;
-                    case 1:
-                        return plugin0;
-                    case 2:
-                        return plugin1;
-                    case 3:
-                        return plugin2;
-                    case 4:
-                        return plugin4;
-                    default:
-                        break;
-                }
-            },
-        },
-        refresh: {
-            get: () => {
-                return refreshValue;
-            },
-            set: (value) => {
-                refreshValue = value;
-            },
-        },
-    });
-
-    Object.defineProperty(Object.getPrototypeOf(navigator), 'plugins', {
-        get: () => {
-            return pluginArray;
-        },
-    });
-})();

server/src/main/resources/cloudflare-js/chrome.runtime.js

@@ -1,170 +0,0 @@
-(function () {
-  window.chrome = {};
-  window.chrome.app = {
-    InstallState: {
-      DISABLED: 'disabled',
-      INSTALLED: 'installed',
-      NOT_INSTALLED: 'not_installed',
-    },
-    RunningState: {
-      CANNOT_RUN: 'cannot_run',
-      READY_TO_RUN: 'ready_to_run',
-      RUNNING: 'running',
-    },
-    getDetails: () => {
-      '[native code]';
-    },
-    getIsInstalled: () => {
-      '[native code]';
-    },
-    installState: () => {
-      '[native code]';
-    },
-    get isInstalled() {
-      return false;
-    },
-    runningState: () => {
-      '[native code]';
-    },
-  };
-
-  window.chrome.runtime = {
-    OnInstalledReason: {
-      CHROME_UPDATE: 'chrome_update',
-      INSTALL: 'install',
-      SHARED_MODULE_UPDATE: 'shared_module_update',
-      UPDATE: 'update',
-    },
-    OnRestartRequiredReason: {
-      APP_UPDATE: 'app_update',
-      OS_UPDATE: 'os_update',
-      PERIODIC: 'periodic',
-    },
-    PlatformArch: {
-      ARM: 'arm',
-      ARM64: 'arm64',
-      MIPS: 'mips',
-      MIPS64: 'mips64',
-      X86_32: 'x86-32',
-      X86_64: 'x86-64',
-    },
-    PlatformNaclArch: {
-      ARM: 'arm',
-      MIPS: 'mips',
-      MIPS64: 'mips64',
-      X86_32: 'x86-32',
-      X86_64: 'x86-64',
-    },
-    PlatformOs: {
-      ANDROID: 'android',
-      CROS: 'cros',
-      FUCHSIA: 'fuchsia',
-      LINUX: 'linux',
-      MAC: 'mac',
-      OPENBSD: 'openbsd',
-      WIN: 'win',
-    },
-    RequestUpdateCheckStatus: {
-      NO_UPDATE: 'no_update',
-      THROTTLED: 'throttled',
-      UPDATE_AVAILABLE: 'update_available',
-    },
-    connect() {
-      '[native code]';
-    },
-    sendMessage() {
-      '[native code]';
-    },
-    id: undefined,
-  };
-
-  let startE = Date.now();
-  window.chrome.csi = function () {
-    '[native code]';
-    return {
-      startE: startE,
-      onloadT: startE + 281,
-      pageT: 3947.235,
-      tran: 15,
-    };
-  };
-
-  window.chrome.loadTimes = function () {
-    '[native code]';
-    return {
-      get requestTime() {
-        return startE / 1000;
-      },
-      get startLoadTime() {
-        return startE / 1000;
-      },
-      get commitLoadTime() {
-        return startE / 1000 + 0.324;
-      },
-      get finishDocumentLoadTime() {
-        return startE / 1000 + 0.498;
-      },
-      get finishLoadTime() {
-        return startE / 1000 + 0.534;
-      },
-      get firstPaintTime() {
-        return startE / 1000 + 0.437;
-      },
-      get firstPaintAfterLoadTime() {
-        return 0;
-      },
-      get navigationType() {
-        return 'Other';
-      },
-      get wasFetchedViaSpdy() {
-        return true;
-      },
-      get wasNpnNegotiated() {
-        return true;
-      },
-      get npnNegotiatedProtocol() {
-        return 'h3';
-      },
-      get wasAlternateProtocolAvailable() {
-        return false;
-      },
-      get connectionInfo() {
-        return 'h3';
-      },
-    };
-  };
-})();
-
-// Bypass OOPIF test
-(function performance_memory() {
-  const jsHeapSizeLimitInt = 4294705152;
-
-  const total_js_heap_size = 35244183;
-  const used_js_heap_size = [
-    17632315, 17632315, 17632315, 17634847, 17636091, 17636751,
-  ];
-
-  let counter = 0;
-
-  let MemoryInfoProto = Object.getPrototypeOf(performance.memory);
-  Object.defineProperties(MemoryInfoProto, {
-    jsHeapSizeLimit: {
-      get: () => {
-        return jsHeapSizeLimitInt;
-      },
-    },
-    totalJSHeapSize: {
-      get: () => {
-        return total_js_heap_size;
-      },
-    },
-    usedJSHeapSize: {
-      get: () => {
-        if (counter > 5) {
-          counter = 0;
-        }
-        return used_js_heap_size[counter++];
-      },
-    },
-  });
-})();

server/src/main/resources/cloudflare-js/emulate.touch.js

@@ -1,3 +0,0 @@
-Object.defineProperty(navigator, 'maxTouchPoints', {
-    get: () => 1
-});

server/src/main/resources/cloudflare-js/navigator.permissions.js

@@ -1,33 +0,0 @@
-// https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js
-if (!window.Notification) {
-    window.Notification = {
-        permission: 'denied'
-    }
-}
-const originalQuery = window.navigator.permissions.query
-window.navigator.permissions.__proto__.query = parameters =>
-    parameters.name === 'notifications'
-        ? Promise.resolve({state: window.Notification.permission})
-        : originalQuery(parameters)
-const oldCall = Function.prototype.call
-
-function call() {
-    return oldCall.apply(this, arguments)
-}
-
-Function.prototype.call = call
-const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
-const oldToString = Function.prototype.toString
-
-function functionToString() {
-    if (this === window.navigator.permissions.query) {
-        return 'function query() { [native code] }'
-    }
-    if (this === functionToString) {
-        return nativeToStringFunctionString
-    }
-    return oldCall.call(oldToString, this)
-}
-
-// eslint-disable-next-line
-Function.prototype.toString = functionToString

server/src/main/resources/cloudflare-js/navigator.webdriver.js

@@ -1,5 +0,0 @@
-Object.defineProperty(Navigator.prototype, 'webdriver', {
-  get() {
-    return false;
-  },
-});

server/src/main/resources/server-reference.conf

@@ -29,3 +29,7 @@ server.basicAuthPassword = ""
 # misc
 server.debugLogsEnabled = false
 server.systemTrayEnabled = true
+
+# Webview
+server.webviewEnabled = false
+server.undetectedChromePath = ""