Switch to a new Ktlint Formatter (#705)

* Switch to new Ktlint plugin

* Add ktlintCheck to PR builds

* Run formatter

* Put ktlint version in libs toml

* Fix lint

* Use Zip4Java from libs.toml
This commit is contained in:
Mitchell Syer
2023-10-06 23:38:39 -04:00
committed by GitHub
parent 3cd3cb0186
commit 849acfca3d
277 changed files with 6709 additions and 5090 deletions

View File

@@ -9,7 +9,6 @@ import kotlinx.coroutines.withContext
* Util for evaluating JavaScript in sources.
*/
class JavaScriptEngine(context: Context) {
/**
* Evaluate arbitrary JavaScript code and get the result as a primitive type
* (e.g., String, Int).
@@ -19,9 +18,10 @@ class JavaScriptEngine(context: Context) {
* @return Result of JavaScript code as a primitive type.
*/
@Suppress("UNUSED", "UNCHECKED_CAST")
suspend fun <T> evaluate(script: String): T = withContext(Dispatchers.IO) {
QuickJs.create().use {
it.evaluate(script) as T
suspend fun <T> evaluate(script: String): T =
withContext(Dispatchers.IO) {
QuickJs.create().use {
it.evaluate(script) as T
}
}
}
}

View File

@@ -33,7 +33,10 @@ class MemoryCookieJar : CookieJar {
}
@Synchronized
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
override fun saveFromResponse(
url: HttpUrl,
cookies: List<Cookie>,
) {
val cookiesToAdd = cookies.map { WrappedCookie.wrap(it) }
cache.removeAll(cookiesToAdd)

View File

@@ -27,8 +27,7 @@ import java.util.concurrent.TimeUnit
@Suppress("UNUSED_PARAMETER")
class NetworkHelper(context: Context) {
// private val preferences: PreferencesHelper by injectLazy()
// private val preferences: PreferencesHelper by injectLazy()
// private val cacheDir = File(context.cacheDir, "network_cache")
@@ -36,31 +35,36 @@ class NetworkHelper(context: Context) {
// Tachidesk -->
val cookieStore = PersistentCookieStore(context)
init {
CookieHandler.setDefault(
CookieManager(cookieStore, CookiePolicy.ACCEPT_ALL)
CookieManager(cookieStore, CookiePolicy.ACCEPT_ALL),
)
}
// Tachidesk <--
private val baseClientBuilder: OkHttpClient.Builder
get() {
val builder = OkHttpClient.Builder()
.cookieJar(PersistentCookieJar(cookieStore))
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.callTimeout(2, TimeUnit.MINUTES)
.addInterceptor(UserAgentInterceptor())
val builder =
OkHttpClient.Builder()
.cookieJar(PersistentCookieJar(cookieStore))
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.callTimeout(2, TimeUnit.MINUTES)
.addInterceptor(UserAgentInterceptor())
val httpLoggingInterceptor = HttpLoggingInterceptor(object : HttpLoggingInterceptor.Logger {
val logger = KotlinLogging.logger { }
val httpLoggingInterceptor =
HttpLoggingInterceptor(
object : HttpLoggingInterceptor.Logger {
val logger = KotlinLogging.logger { }
override fun log(message: String) {
logger.debug { message }
override fun log(message: String) {
logger.debug { message }
}
},
).apply {
level = HttpLoggingInterceptor.Level.BASIC
}
}).apply {
level = HttpLoggingInterceptor.Level.BASIC
}
builder.addInterceptor(httpLoggingInterceptor)
// when (preferences.dohProvider()) {

View File

@@ -25,31 +25,32 @@ fun Call.asObservable(): Observable<Response> {
val call = clone()
// Wrap the call in a helper which handles both unsubscription and backpressure.
val requestArbiter = object : AtomicBoolean(), Producer, Subscription {
override fun request(n: Long) {
if (n == 0L || !compareAndSet(false, true)) return
val requestArbiter =
object : AtomicBoolean(), Producer, Subscription {
override fun request(n: Long) {
if (n == 0L || !compareAndSet(false, true)) return
try {
val response = call.execute()
if (!subscriber.isUnsubscribed) {
subscriber.onNext(response)
subscriber.onCompleted()
}
} catch (error: Exception) {
if (!subscriber.isUnsubscribed) {
subscriber.onError(error)
try {
val response = call.execute()
if (!subscriber.isUnsubscribed) {
subscriber.onNext(response)
subscriber.onCompleted()
}
} catch (error: Exception) {
if (!subscriber.isUnsubscribed) {
subscriber.onError(error)
}
}
}
}
override fun unsubscribe() {
// call.cancel()
}
override fun unsubscribe() {
// call.cancel()
}
override fun isUnsubscribed(): Boolean {
return call.isCanceled()
override fun isUnsubscribed(): Boolean {
return call.isCanceled()
}
}
}
subscriber.add(requestArbiter)
subscriber.setProducer(requestArbiter)
@@ -72,13 +73,19 @@ private suspend fun Call.await(callStack: Array<StackTraceElement>): Response {
return suspendCancellableCoroutine { continuation ->
val callback =
object : Callback {
override fun onResponse(call: Call, response: Response) {
override fun onResponse(
call: Call,
response: Response,
) {
continuation.resume(response) {
response.body.close()
}
}
override fun onFailure(call: Call, e: IOException) {
override fun onFailure(
call: Call,
e: IOException,
) {
// Don't bother with resuming the continuation if it is already cancelled.
if (continuation.isCancelled) return
val exception = IOException(e.message, e).apply { stackTrace = callStack }
@@ -116,16 +123,20 @@ suspend fun Call.awaitSuccess(): Response {
return response
}
fun OkHttpClient.newCachelessCallWithProgress(request: Request, listener: ProgressListener): Call {
val progressClient = newBuilder()
.cache(null)
.addNetworkInterceptor { chain ->
val originalResponse = chain.proceed(chain.request())
originalResponse.newBuilder()
.body(ProgressResponseBody(originalResponse.body, listener))
.build()
}
.build()
fun OkHttpClient.newCachelessCallWithProgress(
request: Request,
listener: ProgressListener,
): Call {
val progressClient =
newBuilder()
.cache(null)
.addNetworkInterceptor { chain ->
val originalResponse = chain.proceed(chain.request())
originalResponse.newBuilder()
.body(ProgressResponseBody(originalResponse.body, listener))
.build()
}
.build()
return progressClient.newCall(request)
}
@@ -139,7 +150,7 @@ context(Json)
@OptIn(ExperimentalSerializationApi::class)
fun <T> decodeFromJsonResponse(
deserializer: DeserializationStrategy<T>,
response: Response
response: Response,
): T {
return response.body.source().use {
decodeFromBufferedSource(deserializer, it)

View File

@@ -6,8 +6,10 @@ import okhttp3.HttpUrl
// from TachiWeb-Server
class PersistentCookieJar(private val store: PersistentCookieStore) : CookieJar {
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
override fun saveFromResponse(
url: HttpUrl,
cookies: List<Cookie>,
) {
store.addAll(url, cookies)
}

View File

@@ -15,7 +15,6 @@ import kotlin.time.Duration.Companion.seconds
// from TachiWeb-Server
class PersistentCookieStore(context: Context) : CookieStore {
private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
@@ -28,8 +27,9 @@ class PersistentCookieStore(context: Context) : CookieStore {
if (cookies != null) {
try {
val url = "http://$key".toHttpUrlOrNull() ?: continue
val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
.filter { !it.hasExpired() }
val nonExpiredCookies =
cookies.mapNotNull { Cookie.parse(url, it) }
.filter { !it.hasExpired() }
cookieMap.put(key, nonExpiredCookies)
} catch (e: Exception) {
// Ignore
@@ -38,7 +38,10 @@ class PersistentCookieStore(context: Context) : CookieStore {
}
}
fun addAll(url: HttpUrl, cookies: List<Cookie>) {
fun addAll(
url: HttpUrl,
cookies: List<Cookie>,
) {
lock.withLock {
val uri = url.toUri()
@@ -75,13 +78,17 @@ class PersistentCookieStore(context: Context) : CookieStore {
}
}
override fun get(uri: URI): List<HttpCookie> = get(uri.host).map {
it.toHttpCookie()
}
override fun get(uri: URI): List<HttpCookie> =
get(uri.host).map {
it.toHttpCookie()
}
fun get(url: HttpUrl) = get(url.toUri().host)
override fun add(uri: URI?, cookie: HttpCookie) {
override fun add(
uri: URI?,
cookie: HttpCookie,
) {
@Suppress("NAME_SHADOWING")
val uri = uri ?: URI("http://" + cookie.domain.removePrefix("."))
lock.withLock {
@@ -105,15 +112,19 @@ class PersistentCookieStore(context: Context) : CookieStore {
}
}
override fun remove(uri: URI?, cookie: HttpCookie): Boolean {
override fun remove(
uri: URI?,
cookie: HttpCookie,
): Boolean {
@Suppress("NAME_SHADOWING")
val uri = uri ?: URI("http://" + cookie.domain.removePrefix("."))
return lock.withLock {
val cookies = cookieMap[uri.host].orEmpty()
val index = cookies.indexOfFirst {
it.name == cookie.name &&
it.path == cookie.path
}
val index =
cookies.indexOfFirst {
it.name == cookie.name &&
it.path == cookie.path
}
if (index >= 0) {
val newList = cookies.toMutableList()
newList.removeAt(index)
@@ -132,45 +143,47 @@ class PersistentCookieStore(context: Context) : CookieStore {
private fun saveToDisk(uri: URI) {
// Get cookies to be stored in disk
val newValues = cookieMap[uri.host]
.orEmpty()
.asSequence()
.filter { it.persistent && !it.hasExpired() }
.map(Cookie::toString)
.toSet()
val newValues =
cookieMap[uri.host]
.orEmpty()
.asSequence()
.filter { it.persistent && !it.hasExpired() }
.map(Cookie::toString)
.toSet()
prefs.edit().putStringSet(uri.host, newValues).apply()
}
private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt
private fun HttpCookie.toCookie(uri: URI) = Cookie.Builder()
.name(name)
.value(value)
.domain(uri.host)
.path(path ?: "/")
.let {
if (maxAge != -1L) {
it.expiresAt(System.currentTimeMillis() + maxAge.seconds.inWholeMilliseconds)
} else {
it.expiresAt(Long.MAX_VALUE)
private fun HttpCookie.toCookie(uri: URI) =
Cookie.Builder()
.name(name)
.value(value)
.domain(uri.host)
.path(path ?: "/")
.let {
if (maxAge != -1L) {
it.expiresAt(System.currentTimeMillis() + maxAge.seconds.inWholeMilliseconds)
} else {
it.expiresAt(Long.MAX_VALUE)
}
}
}
.let {
if (secure) {
it.secure()
} else {
it
.let {
if (secure) {
it.secure()
} else {
it
}
}
}
.let {
if (isHttpOnly) {
it.httpOnly()
} else {
it
.let {
if (isHttpOnly) {
it.httpOnly()
} else {
it
}
}
}
.build()
.build()
private fun Cookie.toHttpCookie(): HttpCookie {
val it = this
@@ -178,11 +191,12 @@ class PersistentCookieStore(context: Context) : CookieStore {
domain = it.domain
path = it.path
secure = it.secure
maxAge = if (it.persistent) {
-1
} else {
(it.expiresAt.milliseconds - System.currentTimeMillis().milliseconds).inWholeSeconds
}
maxAge =
if (it.persistent) {
-1
} else {
(it.expiresAt.milliseconds - System.currentTimeMillis().milliseconds).inWholeSeconds
}
isHttpOnly = it.httpOnly
}

View File

@@ -1,5 +1,9 @@
package eu.kanade.tachiyomi.network
interface ProgressListener {
fun update(bytesRead: Long, contentLength: Long, done: Boolean)
fun update(
bytesRead: Long,
contentLength: Long,
done: Boolean,
)
}

View File

@@ -10,7 +10,6 @@ import okio.buffer
import java.io.IOException
class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
private val bufferedSource: BufferedSource by lazy {
source(responseBody.source()).buffer()
}
@@ -32,7 +31,10 @@ class ProgressResponseBody(private val responseBody: ResponseBody, private val p
var totalBytesRead = 0L
@Throws(IOException::class)
override fun read(sink: Buffer, byteCount: Long): Long {
override fun read(
sink: Buffer,
byteCount: Long,
): Long {
val bytesRead = super.read(sink, byteCount)
// read() returns the number of bytes read, or -1 if this source is exhausted.
totalBytesRead += if (bytesRead != -1L) bytesRead else 0

View File

@@ -1,3 +1,5 @@
@file:Suppress("ktlint:standard:function-naming")
package eu.kanade.tachiyomi.network
import okhttp3.CacheControl
@@ -15,7 +17,7 @@ private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
fun GET(
url: String,
headers: Headers = DEFAULT_HEADERS,
cache: CacheControl = DEFAULT_CACHE_CONTROL
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
@@ -30,7 +32,7 @@ fun GET(
fun GET(
url: HttpUrl,
headers: Headers = DEFAULT_HEADERS,
cache: CacheControl = DEFAULT_CACHE_CONTROL
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
@@ -43,7 +45,7 @@ fun POST(
url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)

View File

@@ -81,51 +81,56 @@ object CFClearance {
logger.debug { "resolveWithWebView($url)" }
val cookies = Playwright.create().use { playwright ->
playwright.chromium().launch(
LaunchOptions()
.setHeadless(false)
.apply {
if (serverConfig.socksProxyEnabled.value) {
setProxy("socks5://${serverConfig.socksProxyHost.value}:${serverConfig.socksProxyPort.value}")
val cookies =
Playwright.create().use { playwright ->
playwright.chromium().launch(
LaunchOptions()
.setHeadless(false)
.apply {
if (serverConfig.socksProxyEnabled.value) {
setProxy("socks5://${serverConfig.socksProxyHost.value}:${serverConfig.socksProxyPort.value}")
}
},
).use { browser ->
val userAgent = originalRequest.header("User-Agent")
if (userAgent != null) {
browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
browserContext.newPage().use { getCookies(it, url) }
}
} else {
browser.newPage().use { getCookies(it, url) }
}
).use { browser ->
val userAgent = originalRequest.header("User-Agent")
if (userAgent != null) {
browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
browserContext.newPage().use { getCookies(it, url) }
}
} else {
browser.newPage().use { getCookies(it, url) }
}
}
}
// Copy cookies to cookie store
cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
network.cookieStore.addAll(
url = HttpUrl.Builder()
.scheme("http")
.host(domain)
.build(),
cookies = cookies
url =
HttpUrl.Builder()
.scheme("http")
.host(domain)
.build(),
cookies = cookies,
)
}
// Merge new and existing cookies for this request
// Find the cookies that we need to merge into this request
val convertedForThisRequest = cookies.filter {
it.matches(originalRequest.url)
}
val convertedForThisRequest =
cookies.filter {
it.matches(originalRequest.url)
}
// Extract cookies from current request
val existingCookies = Cookie.parseAll(
originalRequest.url,
originalRequest.headers
)
val existingCookies =
Cookie.parseAll(
originalRequest.url,
originalRequest.headers,
)
// Filter out existing values of cookies that we are about to merge in
val filteredExisting = existingCookies.filter { existing ->
convertedForThisRequest.none { converted -> converted.name == existing.name }
}
val filteredExisting =
existingCookies.filter { existing ->
convertedForThisRequest.none { converted -> converted.name == existing.name }
}
logger.trace { "Existing cookies" }
logger.trace { existingCookies.joinToString("; ") }
val newCookies = filteredExisting + convertedForThisRequest
@@ -143,7 +148,7 @@ object CFClearance {
Playwright.create().use { playwright ->
playwright.chromium().launch(
LaunchOptions()
.setHeadless(true)
.setHeadless(true),
).use { browser ->
browser.newPage().use { page ->
val userAgent = page.evaluate("() => {return navigator.userAgent}") as String
@@ -158,7 +163,10 @@ object CFClearance {
}
}
private fun getCookies(page: Page, url: String): List<Cookie> {
private fun getCookies(
page: Page,
url: String,
): List<Cookie> {
applyStealthInitScripts(page)
page.navigate(url)
val challengeResolved = waitForChallengeResolve(page)
@@ -198,7 +206,7 @@ object CFClearance {
ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText()
ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText(),
)
}
@@ -215,12 +223,13 @@ object CFClearance {
val timeoutSeconds = 120
repeat(timeoutSeconds) {
page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
val success = try {
page.querySelector("#challenge-form") == null
} catch (e: Exception) {
logger.debug(e) { "query Error" }
false
}
val success =
try {
page.querySelector("#challenge-form") == null
} catch (e: Exception) {
logger.debug(e) { "query Error" }
false
}
if (success) return true
}
return false

View File

@@ -34,7 +34,7 @@ import kotlin.time.toDurationUnit
fun OkHttpClient.Builder.rateLimit(
permits: Int,
period: Long = 1,
unit: TimeUnit = TimeUnit.SECONDS
unit: TimeUnit = TimeUnit.SECONDS,
) = addInterceptor(RateLimitInterceptor(null, permits, period.toDuration(unit.toDurationUnit())))
/**
@@ -50,17 +50,18 @@ fun OkHttpClient.Builder.rateLimit(
* @param permits [Int] Number of requests allowed within a period of units.
* @param period [Duration] The limiting duration. Defaults to 1.seconds.
*/
fun OkHttpClient.Builder.rateLimit(permits: Int, period: Duration = 1.seconds) =
addInterceptor(RateLimitInterceptor(null, permits, period))
fun OkHttpClient.Builder.rateLimit(
permits: Int,
period: Duration = 1.seconds,
) = addInterceptor(RateLimitInterceptor(null, permits, period))
/** We can probably accept domains or wildcards by comparing with [endsWith], etc. */
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
internal class RateLimitInterceptor(
private val host: String?,
private val permits: Int,
period: Duration
period: Duration,
) : Interceptor {
private val requestQueue = ArrayDeque<Long>(permits)
private val rateLimitMillis = period.inWholeMilliseconds
private val fairLock = Semaphore(1, true)
@@ -98,7 +99,8 @@ internal class RateLimitInterceptor(
} else if (hasRemovedExpired) {
break
} else {
try { // wait for the first entry to expire, or notified by cached response
try {
// wait for the first entry to expire, or notified by cached response
(requestQueue as Object).wait(requestQueue.first - periodStart)
} catch (_: InterruptedException) {
continue

View File

@@ -29,7 +29,7 @@ fun OkHttpClient.Builder.rateLimitHost(
httpUrl: HttpUrl,
permits: Int,
period: Long = 1,
unit: TimeUnit = TimeUnit.SECONDS
unit: TimeUnit = TimeUnit.SECONDS,
) = addInterceptor(RateLimitInterceptor(httpUrl.host, permits, period.toDuration(unit.toDurationUnit())))
/**
@@ -49,7 +49,7 @@ fun OkHttpClient.Builder.rateLimitHost(
fun OkHttpClient.Builder.rateLimitHost(
httpUrl: HttpUrl,
permits: Int,
period: Duration = 1.seconds
period: Duration = 1.seconds,
): OkHttpClient.Builder = addInterceptor(RateLimitInterceptor(httpUrl.host, permits, period))
/**
@@ -69,5 +69,5 @@ fun OkHttpClient.Builder.rateLimitHost(
fun OkHttpClient.Builder.rateLimitHost(
url: String,
permits: Int,
period: Duration = 1.seconds
period: Duration = 1.seconds,
): OkHttpClient.Builder = addInterceptor(RateLimitInterceptor(url.toHttpUrlOrNull()?.host, permits, period))

View File

@@ -9,11 +9,12 @@ class UserAgentInterceptor : Interceptor {
val originalRequest = chain.request()
return if (originalRequest.header("User-Agent").isNullOrEmpty()) {
val newRequest = originalRequest
.newBuilder()
.removeHeader("User-Agent")
.addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
.build()
val newRequest =
originalRequest
.newBuilder()
.removeHeader("User-Agent")
.addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
.build()
chain.proceed(newRequest)
} else {
chain.proceed(originalRequest)