Implement FlareSolverr (#844)

* Implement FlareSolverr

* Oops

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -7,30 +7,29 @@ package eu.kanade.tachiyomi.network
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-// import android.content.Context
-// import eu.kanade.tachiyomi.BuildConfig
-// import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-// import okhttp3.HttpUrl.Companion.toHttpUrl
-// import okhttp3.dnsoverhttps.DnsOverHttps
-// import okhttp3.logging.HttpLoggingInterceptor
-// import uy.kohesive.injekt.injectLazy
 import android.content.Context
 import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
 import eu.kanade.tachiyomi.network.interceptor.IgnoreGzipInterceptor
 import eu.kanade.tachiyomi.network.interceptor.UncaughtExceptionInterceptor
 import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.drop
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
 import mu.KotlinLogging
 import okhttp3.Cache
 import okhttp3.OkHttpClient
 import okhttp3.brotli.BrotliInterceptor
 import okhttp3.logging.HttpLoggingInterceptor
+import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
 import java.io.File
 import java.net.CookieHandler
 import java.net.CookieManager
 import java.net.CookiePolicy
 import java.util.concurrent.TimeUnit
 
-@Suppress("UNUSED_PARAMETER")
 class NetworkHelper(context: Context) {
     //    private val preferences: PreferencesHelper by injectLazy()
 
@@ -48,6 +47,26 @@ class NetworkHelper(context: Context) {
     }
     // Tachidesk <--
 
+    private val userAgent =
+        MutableStateFlow(
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        )
+
+    fun defaultUserAgentProvider(): String {
+        return userAgent.value
+    }
+
+    init {
+        @OptIn(DelicateCoroutinesApi::class)
+        userAgent
+            .drop(1)
+            .onEach {
+                GetCatalogueSource.unregisterAllCatalogueSources() // need to reset the headers
+            }
+            .launchIn(GlobalScope)
+    }
+
     private val baseClientBuilder: OkHttpClient.Builder
         get() {
             val builder =
@@ -63,7 +82,7 @@ class NetworkHelper(context: Context) {
                         ),
                     )
                     .addInterceptor(UncaughtExceptionInterceptor())
-                    .addInterceptor(UserAgentInterceptor())
+                    .addInterceptor(UserAgentInterceptor(::defaultUserAgentProvider))
                     .addNetworkInterceptor(IgnoreGzipInterceptor())
                     .addNetworkInterceptor(BrotliInterceptor)
 
@@ -78,14 +97,14 @@ class NetworkHelper(context: Context) {
                         }
                     },
                 ).apply {
-                    level = HttpLoggingInterceptor.Level.BASIC
+                    level = HttpLoggingInterceptor.Level.HEADERS
                 }
             builder.addNetworkInterceptor(httpLoggingInterceptor)
             // }
 
-            // builder.addInterceptor(
+            builder.addInterceptor(
-            //     CloudflareInterceptor(context, cookieJar, ::defaultUserAgentProvider),
+                CloudflareInterceptor(setUserAgent = { userAgent.value = it }),
-            // )
+            )
 
             // when (preferences.dohProvider().get()) {
             //     PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
@@ -108,9 +127,5 @@ class NetworkHelper(context: Context) {
 //    val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
     val client by lazy { baseClientBuilder.build() }
 
-    val cloudflareClient by lazy {
+    val cloudflareClient by lazy { client }
-        client.newBuilder()
-            .addInterceptor(CloudflareInterceptor())
-            .build()
-    }
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -21,16 +21,18 @@ class PersistentCookieStore(context: Context) : CookieStore {
     private val lock = ReentrantLock()
 
     init {
-        for ((key, value) in prefs.all) {
+        val domains =
-            @Suppress("UNCHECKED_CAST")
+            prefs.all.keys.map { it.substringBeforeLast(".") }
-            val cookies = value as? Set<String>
+                .toSet()
-            if (cookies != null) {
+        domains.forEach { domain ->
+            val cookies = prefs.getStringSet(domain, emptySet())
+            if (!cookies.isNullOrEmpty()) {
                 try {
-                    val url = "http://$key".toHttpUrlOrNull() ?: continue
+                    val url = "http://$domain".toHttpUrlOrNull() ?: return@forEach
                     val nonExpiredCookies =
                         cookies.mapNotNull { Cookie.parse(url, it) }
                             .filter { !it.hasExpired() }
-                    cookieMap.put(key, nonExpiredCookies)
+                    cookieMap[domain] = nonExpiredCookies
                 } catch (e: Exception) {
                     // Ignore
                 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -1,39 +1,62 @@
 package eu.kanade.tachiyomi.network.interceptor
 
 import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.awaitSuccess
+import eu.kanade.tachiyomi.network.parseAs
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import mu.KotlinLogging
+import okhttp3.Cookie
+import okhttp3.HttpUrl
 import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.IOException
+import kotlin.time.Duration.Companion.seconds
 
-class CloudflareInterceptor : Interceptor {
+class CloudflareInterceptor(
+    private val setUserAgent: (String) -> Unit,
+) : Interceptor {
     private val logger = KotlinLogging.logger {}
 
     private val network: NetworkHelper by injectLazy()
 
-    @Suppress("UNUSED_VARIABLE", "UNREACHABLE_CODE")
     override fun intercept(chain: Interceptor.Chain): Response {
         val originalRequest = chain.request()
 
         logger.trace { "CloudflareInterceptor is being used." }
 
-        val originalResponse = chain.proceed(chain.request())
+        val originalResponse = chain.proceed(originalRequest)
 
         // Check if Cloudflare anti-bot is on
         if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) {
             return originalResponse
         }
 
+        if (!serverConfig.flareSolverrEnabled.value) {
             throw IOException("Cloudflare bypass currently disabled")
+        }
 
         logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
 
         return try {
             originalResponse.close()
-            network.cookieStore.remove(originalRequest.url.toUri())
+            // network.cookieStore.remove(originalRequest.url.toUri())
 
-            val request = originalRequest // resolveWithWebView(originalRequest)
+            val request =
+                runBlocking {
+                    CFClearance.resolveWithFlareSolverr(setUserAgent, originalRequest)
+                }
 
             chain.proceed(request)
         } catch (e: Exception) {
@@ -57,172 +80,140 @@ class CloudflareInterceptor : Interceptor {
 object CFClearance {
     private val logger = KotlinLogging.logger {}
     private val network: NetworkHelper by injectLazy()
+    private val json: Json by injectLazy()
+    private val jsonMediaType = "application/json".toMediaType()
+    private val mutex = Mutex()
 
-    /*init {
+    @Serializable
-        // Fix the default DriverJar issue by providing our own implementation
+    data class FlareSolverCookie(
-        // ref: https://github.com/microsoft/playwright-java/issues/1138
+        val name: String,
-        System.setProperty("playwright.driver.impl", "suwayomi.tachidesk.server.util.DriverJar")
+        val value: String,
-    }
+    )
 
-    fun resolveWithWebView(originalRequest: Request): Request {
+    @Serializable
-        val url = originalRequest.url.toString()
+    data class FlareSolverRequest(
+        val cmd: String,
+        val url: String,
+        val maxTimeout: Int? = null,
+        val session: List<String>? = null,
+        @SerialName("session_ttl_minutes")
+        val sessionTtlMinutes: Int? = null,
+        val cookies: List<FlareSolverCookie>? = null,
+        val returnOnlyCookies: Boolean? = null,
+        val proxy: String? = null,
+        val postData: String? = null, // only used with cmd 'request.post'
+    )
 
-        logger.debug { "resolveWithWebView($url)" }
+    @Serializable
+    data class FlareSolverSolutionCookie(
+        val name: String,
+        val value: String,
+        val domain: String,
+        val path: String,
+        val expires: Double? = null,
+        val size: Int? = null,
+        val httpOnly: Boolean,
+        val secure: Boolean,
+        val session: Boolean? = null,
+        val sameSite: String,
+    )
 
-        val cookies =
+    @Serializable
-            Playwright.create().use { playwright ->
+    data class FlareSolverSolution(
-                playwright.chromium().launch(
+        val url: String,
-                    LaunchOptions()
+        val status: Int,
-                        .setHeadless(false)
+        val headers: Map<String, String>? = null,
-                        .apply {
+        val response: String? = null,
-                            if (serverConfig.socksProxyEnabled.value) {
+        val cookies: List<FlareSolverSolutionCookie>,
-                                setProxy("socks5://${serverConfig.socksProxyHost.value}:${serverConfig.socksProxyPort.value}")
+        val userAgent: String,
-                            }
+    )
+
+    @Serializable
+    data class FlareSolverResponse(
+        val solution: FlareSolverSolution,
+        val status: String,
+        val message: String,
+        val startTimestamp: Long,
+        val endTimestamp: Long,
+        val version: String,
+    )
+
+    suspend fun resolveWithFlareSolverr(
+        setUserAgent: (String) -> Unit,
+        originalRequest: Request,
+    ): Request {
+        val flareSolverResponse =
+            with(json) {
+                mutex.withLock {
+                    network.client.newCall(
+                        POST(
+                            url = serverConfig.flareSolverrUrl.value.removeSuffix("/") + "/v1",
+                            body =
+                                Json.encodeToString(
+                                    FlareSolverRequest(
+                                        "request.get",
+                                        originalRequest.url.toString(),
+                                        cookies =
+                                            network.cookieStore.get(originalRequest.url).map {
+                                                FlareSolverCookie(it.name, it.value)
                                             },
-                ).use { browser ->
+                                        returnOnlyCookies = true,
-                    val userAgent = originalRequest.header("User-Agent")
+                                        maxTimeout =
-                    if (userAgent != null) {
+                                            serverConfig.flareSolverrTimeout.value
-                        browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
+                                                .seconds
-                            browserContext.newPage().use { getCookies(it, url) }
+                                                .inWholeMilliseconds
-                        }
+                                                .toInt(),
-                    } else {
+                                    ),
-                        browser.newPage().use { getCookies(it, url) }
+                                ).toRequestBody(jsonMediaType),
-                    }
+                        ),
+                    ).awaitSuccess().parseAs<FlareSolverResponse>()
                 }
             }
 
-        // Copy cookies to cookie store
+        if (flareSolverResponse.solution.status in 200..299) {
-        cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
+            setUserAgent(flareSolverResponse.solution.userAgent)
+            val cookies =
+                flareSolverResponse.solution.cookies
+                    .map { cookie ->
+                        Cookie.Builder()
+                            .name(cookie.name)
+                            .value(cookie.value)
+                            .domain(cookie.domain)
+                            .path(cookie.path)
+                            .expiresAt(cookie.expires?.takeUnless { it < 0.0 }?.toLong() ?: Long.MAX_VALUE)
+                            .also {
+                                if (cookie.httpOnly) it.httpOnly()
+                                if (cookie.secure) it.secure()
+                            }
+                            .build()
+                    }
+                    .groupBy { it.domain }
+                    .flatMap { (domain, cookies) ->
                         network.cookieStore.addAll(
-                url =
                             HttpUrl.Builder()
                                 .scheme("http")
-                        .host(domain)
+                                .host(domain.removePrefix("."))
                                 .build(),
-                cookies = cookies,
+                            cookies,
                         )
+
+                        cookies
                     }
-        // Merge new and existing cookies for this request
+            logger.trace { "New cookies\n${cookies.joinToString("; ")}" }
-        // Find the cookies that we need to merge into this request
+            val finalCookies =
-        val convertedForThisRequest =
+                network.cookieStore.get(originalRequest.url).joinToString("; ", postfix = "; ") {
-            cookies.filter {
+                    "${it.name}=${it.value}"
-                it.matches(originalRequest.url)
                 }
-        // Extract cookies from current request
+            logger.trace { "Final cookies\n$finalCookies" }
-        val existingCookies =
-            Cookie.parseAll(
-                originalRequest.url,
-                originalRequest.headers,
-            )
-        // Filter out existing values of cookies that we are about to merge in
-        val filteredExisting =
-            existingCookies.filter { existing ->
-                convertedForThisRequest.none { converted -> converted.name == existing.name }
-            }
-        logger.trace { "Existing cookies" }
-        logger.trace { existingCookies.joinToString("; ") }
-        val newCookies = filteredExisting + convertedForThisRequest
-        logger.trace { "New cookies" }
-        logger.trace { newCookies.joinToString("; ") }
             return originalRequest.newBuilder()
-            .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" })
+                .header("Cookie", finalCookies)
+                .header("User-Agent", flareSolverResponse.solution.userAgent)
                 .build()
-    }*/
-
-    fun getWebViewUserAgent(): String {
-        return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-        /*return try {
-            throw PlaywrightException("playwrite is diabled for v0.6.7")
-
-            Playwright.create().use { playwright ->
-                playwright.chromium().launch(
-                    LaunchOptions()
-                        .setHeadless(true),
-                ).use { browser ->
-                    browser.newPage().use { page ->
-                        val userAgent = page.evaluate("() => {return navigator.userAgent}") as String
-                        logger.debug { "WebView User-Agent is $userAgent" }
-                        return userAgent
-                    }
-                }
-            }
-        } catch (e: PlaywrightException) {
-            // Playwright might fail on headless environments like docker
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
-        }*/
-    }
-
-    /*private fun getCookies(
-        page: Page,
-        url: String,
-    ): List<Cookie> {
-        applyStealthInitScripts(page)
-        page.navigate(url)
-        val challengeResolved = waitForChallengeResolve(page)
-
-        return if (challengeResolved) {
-            val cookies = page.context().cookies()
-
-            logger.debug {
-                val userAgent = page.evaluate("() => {return navigator.userAgent}")
-                "Playwright User-Agent is $userAgent"
-            }
-
-            // Convert PlayWright cookies to OkHttp cookies
-            cookies.map {
-                Cookie.Builder()
-                    .domain(it.domain.removePrefix("."))
-                    .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE)
-                    .name(it.name)
-                    .path(it.path)
-                    .value(it.value).apply {
-                        if (it.httpOnly) httpOnly()
-                        if (it.secure) secure()
-                    }.build()
-            }
         } else {
             logger.debug { "Cloudflare challenge failed to resolve" }
             throw CloudflareBypassException()
         }
     }
 
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18
-    private val stealthInitScripts by lazy {
-        arrayOf(
-            ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText(),
-        )
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76
-    private fun applyStealthInitScripts(page: Page) {
-        for (script in stealthInitScripts) {
-            page.addInitScript(script)
-        }
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21
-    private fun waitForChallengeResolve(page: Page): Boolean {
-        // sometimes the user has to solve the captcha challenge manually, potentially wait a long time
-        val timeoutSeconds = 120
-        repeat(timeoutSeconds) {
-            page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
-            val success =
-                try {
-                    page.querySelector("#challenge-form") == null
-                } catch (e: Exception) {
-                    logger.debug(e) { "query Error" }
-                    false
-                }
-            if (success) return true
-        }
-        return false
-    }*/
-
     private class CloudflareBypassException : Exception()
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt

@@ -1,10 +1,9 @@
 package eu.kanade.tachiyomi.network.interceptor
 
-import eu.kanade.tachiyomi.source.online.HttpSource
 import okhttp3.Interceptor
 import okhttp3.Response
 
-class UserAgentInterceptor : Interceptor {
+class UserAgentInterceptor(private val userAgentProvider: () -> String) : Interceptor {
     override fun intercept(chain: Interceptor.Chain): Response {
         val originalRequest = chain.request()
 
@@ -13,7 +12,7 @@ class UserAgentInterceptor : Interceptor {
                 originalRequest
                     .newBuilder()
                     .removeHeader("User-Agent")
-                    .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
+                    .addHeader("User-Agent", userAgentProvider())
                     .build()
             chain.proceed(newRequest)
         } else {

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -4,7 +4,6 @@ import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.network.awaitSuccess
-import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent
 import eu.kanade.tachiyomi.network.newCachelessCallWithProgress
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -107,7 +106,7 @@ abstract class HttpSource : CatalogueSource {
      */
     protected open fun headersBuilder() =
         Headers.Builder().apply {
-            add("User-Agent", DEFAULT_USER_AGENT)
+            add("User-Agent", network.defaultUserAgentProvider())
         }
 
     /**
@@ -480,10 +479,6 @@ abstract class HttpSource : CatalogueSource {
      * Returns the list of filters for the source.
      */
     override fun getFilterList() = FilterList()
-
-    companion object {
-        val DEFAULT_USER_AGENT by lazy { getWebViewUserAgent() }
-    }
 }
 
 class LicensedMangaChaptersException : Exception("Licensed - No chapters to show")

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt

@@ -81,4 +81,8 @@ object GetCatalogueSource {
     fun unregisterCatalogueSource(sourceId: Long) {
         sourceCache.remove(sourceId)
     }
+
+    fun unregisterAllCatalogueSources() {
+        sourceCache.clear()
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -131,6 +131,11 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
     // local source
     val localSourcePath: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)
 
+    // cloudflare bypass
+    val flareSolverrEnabled: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
+    val flareSolverrUrl: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)
+    val flareSolverrTimeout: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
+
     @OptIn(ExperimentalCoroutinesApi::class)
     fun <T> subscribeTo(
         flow: Flow<T>,

server/src/main/resources/server-reference.conf

@@ -56,3 +56,8 @@ server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - d
 
 # local source
 server.localSourcePath = ""
+
+# Cloudflare bypass
+server.flareSolverrEnabled = false
+server.flareSolverrUrl = "http://localhost:8191"
+server.flareSolverrTimeout = 60 # time in seconds