diff --git a/CHANGELOG.md b/CHANGELOG.md index dda51cf96..82c9d38fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added - (**Sync**) Added [SyncYomi](https://github.com/syncyomi/syncyomi) support - (**OPDS**) Add option to skip chapter metadata feed providing direct stream/download links +- (**Extension/API**) Support Extensions API v1.6 ### Changed - (**Database/H2**) Use the latest H2 database engine diff --git a/docs/Configuring-Suwayomi‐Server.md b/docs/Configuring-Suwayomi‐Server.md index 86d601b7d..053a07274 100644 --- a/docs/Configuring-Suwayomi‐Server.md +++ b/docs/Configuring-Suwayomi‐Server.md @@ -159,15 +159,20 @@ server.systemTrayEnabled = true server.maxLogFiles = 31 server.maxLogFileSize = "10mb" server.maxLogFolderSize = "100mb" -server.extensionRepos = [] -server.maxSourcesInParallel = 6 + ``` - `server.debugLogsEnabled` controls whether if Suwayomi-Server should print more information while being run inside a Terminal/CMD/Powershell window. - `server.systemTrayEnabled = true` whether if Suwayomi-Server should show a System Tray Icon, disabling this on headless servers is recommended. - `server.maxLogFiles = 31` sets the maximum number of days to keep files before they get deleted. - `server.maxLogFileSize = "10mb"` sets the maximum size of a log file - values are formatted like: 1 (bytes), 1KB (kilobytes), 1MB (megabytes), 1GB (gigabytes) - `server.maxLogFolderSize = "100mb"` sets the maximum size of all saved log files - values are formatted like: 1 (bytes), 1KB (kilobytes), 1MB (megabytes), 1GB (gigabytes) -- `server.extensionRepos` is a list of extension repositories for custom sources. Uses the same format as Mihon; each entry is expected to be a string URL pointing to a JSON file representing the repository. + +### Extension/Source +``` +server.extensionStores = [] +server.maxSourcesInParallel = 6 +``` +- `server.extensionStores` is a list of extension stores (previously called repositories) for custom sources. Uses the same format as Mihon; each entry is expected to be a string URL pointing to a JSON or PROTOBUF file representing the repository. - `server.maxSourcesInParallel = 6` sets how many sources can do requests (updates, downloads) in parallel. Updates/downloads are grouped by source and all mangas of a source are updated/downloaded synchronously. Range: 1 <= n <= 20. ### Backup diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index afef2f1d4..7190dbb08 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -47,6 +47,7 @@ okhttp-core = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } okhttp-logging = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" } okhttp-dnsoverhttps = { module = "com.squareup.okhttp3:okhttp-dnsoverhttps", version.ref = "okhttp" } okhttp-brotli = { module = "com.squareup.okhttp3:okhttp-brotli", version.ref = "okhttp" } +okhttp-zstd = { module = "com.squareup.okhttp3:okhttp-zstd", version.ref = "okhttp" } okio = "com.squareup.okio:okio:3.17.0" # Javalin api @@ -70,6 +71,7 @@ exposed-dao = { module = "org.jetbrains.exposed:exposed-dao", version.ref = "exp exposed-jdbc = { module = "org.jetbrains.exposed:exposed-jdbc", version.ref = "exposed" } exposed-javatime = { module = "org.jetbrains.exposed:exposed-java-time", version.ref = "exposed" } exposed-kotlintime = { module = "org.jetbrains.exposed:exposed-kotlin-datetime", version.ref = "exposed" } +exposed-json = { module = "org.jetbrains.exposed:exposed-json ", version.ref = "exposed" } postgres = "org.postgresql:postgresql:42.7.11" h2 = "com.h2database:h2:2.4.240" hikaricp = "com.zaxxer:HikariCP:7.1.0" @@ -227,6 +229,7 @@ okhttp = [ "okhttp-logging", "okhttp-dnsoverhttps", "okhttp-brotli", + "okhttp-zstd", ] javalin = [ "javalin-core", @@ -245,6 +248,7 @@ exposed = [ "exposed-jdbc", "exposed-javatime", "exposed-kotlintime", + "exposed-json", ] systemtray = [ "systemtray-core", diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index ce622f1e7..36ece668c 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -276,30 +276,38 @@ class ServerConfig( description = "Ignore re-uploaded chapters from auto-download", ) - val extensionRepos: MutableStateFlow> by ListSetting( + @Deprecated("Will get removed", replaceWith = ReplaceWith("extensionStores")) + val extensionRepos: MutableStateFlow> by MigratedConfigValue( protoNumber = 22, group = SettingGroup.EXTENSION, privacySafe = false, defaultValue = emptyList(), - itemValidator = { url -> - if (url.matches(repoMatchRegex)) { - null - } else { - "Invalid repository URL format" - } - }, - itemToValidValue = { url -> - if (url.matches(repoMatchRegex)) { - url - } else { - null - } - }, + deprecated = + SettingsRegistry.SettingDeprecated( + message = "Replaced with addExtensionStore and removeExtensionStore mutations", + migrateConfigValue = { + @Suppress("UNCHECKED_CAST") + (it.unwrapped() as? List) + ?.map { + if (it.contains("github.com")) { + it.replace(repoMatchRegex) { + "https://raw.githubusercontent.com/${it.groupValues[2]}/${it.groupValues[3]}/" + + (it.groupValues.getOrNull(4)?.ifBlank { null } ?: "repo") + + "/" + + (it.groupValues.getOrNull(5)?.ifBlank { null } ?: "index.min.json") + } + } else { + it + } + } + }, + ), + readMigrated = { extensionStores.value }, + setMigrated = { extensionStores.value = it.distinct() }, typeInfo = SettingsRegistry.PartialTypeInfo( specificType = "List", ), - description = "example: [\"https://github.com/MY_ACCOUNT/MY_REPO/tree/repo\"]", ) val maxSourcesInParallel: MutableStateFlow by IntSetting( @@ -1104,7 +1112,29 @@ class ServerConfig( privacySafe = true, defaultValue = false, description = "Skips the metadata feed and provides download/stream links directly in the chapter list. Improves compatibility with KOReader auto-downloader. KoSync strategies are applied, but PROMPT conflicts are ignored (treating local progress as priority)." + ) + val extensionStores: MutableStateFlow> by ListSetting( + protoNumber = 97, + group = SettingGroup.EXTENSION, + privacySafe = true, + defaultValue = emptyList(), + requiresRestart = true, + itemValidator = { url -> + if (url.isNotEmpty()) { + null + } else { + "Invalid store URL format" + } + }, + itemToValidValue = { url -> + url.ifEmpty { null } + }, + typeInfo = + SettingsRegistry.PartialTypeInfo( + specificType = "List", + ), + description = "List of extension store index URLs", ) /** ****************************************************************** **/ diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/HttpException.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/HttpException.kt new file mode 100644 index 000000000..0b551a656 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/HttpException.kt @@ -0,0 +1,15 @@ +package eu.kanade.tachiyomi.network + +import okhttp3.Response + +/** + * Exception that handles HTTP codes considered not successful by OkHttp. + * Use it to have a standardized error message in the app across the extensions. + * + * @see Response.isSuccessful + * @since tachiyomix 1.6 + * @param code [Int] the HTTP status code + */ +class HttpException( + val code: Int, +) : IllegalStateException("HTTP error $code") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt index ac2e7b0c5..6356e78b3 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt @@ -2,26 +2,25 @@ package eu.kanade.tachiyomi.network import android.content.Context import app.cash.quickjs.QuickJs -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext +import eu.kanade.tachiyomi.util.lang.withIOContext /** * Util for evaluating JavaScript in sources. */ +@Suppress("UNUSED", "UNCHECKED_CAST") class JavaScriptEngine( - @Suppress("UNUSED_PARAMETER") context: Context, + context: Context, ) { /** * Evaluate arbitrary JavaScript code and get the result as a primitive type * (e.g., String, Int). * - * @since extensions-lib 1.4 + * @since tachiyomix 1.4 * @param script JavaScript to execute. * @return Result of JavaScript code as a primitive type. */ - @Suppress("UNUSED", "UNCHECKED_CAST") suspend fun evaluate(script: String): T = - withContext(Dispatchers.IO) { + withIOContext { QuickJs.create().use { it.evaluate(script) as T } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index 503bb1b56..9b6b19bce 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -9,7 +9,6 @@ package eu.kanade.tachiyomi.network import android.content.Context import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor -import eu.kanade.tachiyomi.network.interceptor.IgnoreGzipInterceptor import eu.kanade.tachiyomi.network.interceptor.UncaughtExceptionInterceptor import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor import io.github.oshai.kotlinlogging.KotlinLogging @@ -22,7 +21,6 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import okhttp3.Cache import okhttp3.OkHttpClient -import okhttp3.brotli.BrotliInterceptor import okhttp3.logging.HttpLoggingInterceptor import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource import java.net.CookieHandler @@ -84,8 +82,6 @@ class NetworkHelper( ), ).addInterceptor(UncaughtExceptionInterceptor()) .addInterceptor(UserAgentInterceptor(::defaultUserAgentProvider)) - .addNetworkInterceptor(IgnoreGzipInterceptor()) - .addNetworkInterceptor(BrotliInterceptor) // if (preferences.verboseLogging().get()) { val httpLoggingInterceptor = @@ -128,5 +124,7 @@ class NetworkHelper( // val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() } val client by lazy { baseClientBuilder.build() } + @Deprecated("The regular client handles Cloudflare by default") + @Suppress("UNUSED") val cloudflareClient by lazy { client } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt index 8e6e3f700..4622c67f2 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt @@ -15,11 +15,14 @@ import rx.Observable import rx.Producer import rx.Subscription import java.io.IOException -import java.util.concurrent.atomic.AtomicBoolean +import kotlin.concurrent.atomics.AtomicBoolean +import kotlin.concurrent.atomics.ExperimentalAtomicApi import kotlin.coroutines.resumeWithException val jsonMime = "application/json; charset=utf-8".toMediaType() +@OptIn(ExperimentalAtomicApi::class) +@Deprecated("Use suspend APIs instead") fun Call.asObservable(): Observable { return Observable.unsafeCreate { subscriber -> // Since Call is a one-shot type, clone it for each new subscriber. @@ -27,9 +30,11 @@ fun Call.asObservable(): Observable { // Wrap the call in a helper which handles both unsubscription and backpressure. val requestArbiter = - object : AtomicBoolean(), Producer, Subscription { + object : Producer, Subscription { + val boolean = AtomicBoolean(false) + override fun request(n: Long) { - if (n == 0L || !compareAndSet(false, true)) return + if (n == 0L || !boolean.compareAndSet(expectedValue = false, newValue = true)) return try { val response = call.execute() @@ -37,15 +42,15 @@ fun Call.asObservable(): Observable { subscriber.onNext(response) subscriber.onCompleted() } - } catch (error: Exception) { + } catch (e: Exception) { if (!subscriber.isUnsubscribed) { - subscriber.onError(error) + subscriber.onError(e) } } } override fun unsubscribe() { - // call.cancel() + call.cancel() } override fun isUnsubscribed(): Boolean = call.isCanceled() @@ -56,50 +61,50 @@ fun Call.asObservable(): Observable { } } -fun Call.asObservableSuccess(): Observable = - asObservable() - .doOnNext { response -> - if (!response.isSuccessful) { - response.close() - throw HttpException(response.code) +@Deprecated("Use suspend APIs instead") +fun Call.asObservableSuccess(): Observable { + @Suppress("DEPRECATION") + return asObservable().doOnNext { response -> + if (!response.isSuccessful) { + response.close() + throw HttpException(response.code) + } + } +} + +// Based on https://github.com/square/okhttp/blob/master/okhttp-coroutines/src/main/kotlin/okhttp3/coroutines/ExecuteAsync.kt +// and https://github.com/gildor/kotlin-coroutines-okhttp +private suspend fun Call.await(callStack: Array): Response { + return suspendCancellableCoroutine { continuation -> + continuation.invokeOnCancellation { + try { + this.cancel() + } catch (_: Throwable) { + // ignore } } -// Based on https://github.com/gildor/kotlin-coroutines-okhttp -private suspend fun Call.await(callStack: Array): Response { - return suspendCancellableCoroutine { continuation -> - val callback = + this.enqueue( object : Callback { - override fun onResponse( - call: Call, - response: Response, - ) { - continuation.resume(response) { _, resourceToClose, _ -> - response.body.close() - resourceToClose.close() - } - } - override fun onFailure( call: Call, e: IOException, ) { - // Don't bother with resuming the continuation if it is already cancelled. if (continuation.isCancelled) return val exception = IOException(e.message, e).apply { stackTrace = callStack } continuation.resumeWithException(exception) } - } - enqueue(callback) - - continuation.invokeOnCancellation { - try { - cancel() - } catch (ex: Throwable) { - // Ignore cancel exception - } - } + override fun onResponse( + call: Call, + response: Response, + ) { + continuation.resume(response) { _, value, _ -> + value.close() + } + } + }, + ) } } @@ -109,7 +114,7 @@ suspend fun Call.await(): Response { } /** - * @since extensions-lib 1.5 + * Similar to [await] but throws [HttpException] if [Response.isSuccessful] returns false */ suspend fun Call.awaitSuccess(): Response { val callStack = Exception().stackTrace.run { copyOfRange(1, size) } @@ -150,7 +155,3 @@ fun decodeFromJsonResponse( response.body.source().use { json.decodeFromBufferedSource(deserializer, it) } - -class HttpException( - val code: Int, -) : IllegalStateException("HTTP error $code") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt index 360967733..abc75ced1 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt @@ -35,7 +35,11 @@ class ProgressResponseBody( val bytesRead = super.read(sink, byteCount) // read() returns the number of bytes read, or -1 if this source is exhausted. totalBytesRead += if (bytesRead != -1L) bytesRead else 0 - progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L) + progressListener.update( + totalBytesRead, + responseBody.contentLength(), + bytesRead == -1L, + ) return bytesRead } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt index d384dc571..3b3c5ac8b 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt @@ -6,6 +6,7 @@ import okhttp3.CacheControl import okhttp3.FormBody import okhttp3.Headers import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request import okhttp3.RequestBody import java.util.concurrent.TimeUnit.MINUTES @@ -18,13 +19,7 @@ fun GET( url: String, headers: Headers = DEFAULT_HEADERS, cache: CacheControl = DEFAULT_CACHE_CONTROL, -): Request = - Request - .Builder() - .url(url) - .headers(headers) - .cacheControl(cache) - .build() +): Request = GET(url.toHttpUrl(), headers, cache) /** * @since extensions-lib 1.4 diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/IgnoreGzipInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/IgnoreGzipInterceptor.kt deleted file mode 100644 index f1331a576..000000000 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/IgnoreGzipInterceptor.kt +++ /dev/null @@ -1,21 +0,0 @@ -package eu.kanade.tachiyomi.network.interceptor - -import okhttp3.Interceptor -import okhttp3.Response - -/** - * To use [okhttp3.brotli.BrotliInterceptor] as a network interceptor, - * add [IgnoreGzipInterceptor] right before it. - * - * This nullifies the transparent gzip of [okhttp3.internal.http.BridgeInterceptor] - * so gzip and Brotli are explicitly handled by the [okhttp3.brotli.BrotliInterceptor]. - */ -class IgnoreGzipInterceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - var request = chain.request() - if (request.header("Accept-Encoding") == "gzip") { - request = request.newBuilder().removeHeader("Accept-Encoding").build() - } - return chain.proceed(request) - } -} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/CatalogueSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/CatalogueSource.kt index fa6f75dc1..b24123239 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/CatalogueSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/CatalogueSource.kt @@ -2,6 +2,12 @@ package eu.kanade.tachiyomi.source import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.SMangaUpdate +import kotlinx.coroutines.async +import kotlinx.coroutines.supervisorScope import rx.Observable import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle @@ -11,68 +17,62 @@ interface CatalogueSource : Source { */ override val lang: String - /** - * Whether the source has support for latest updates. - */ - val supportsLatest: Boolean - - /** - * Get a page with a list of manga. - * - * @since extensions-lib 1.5 - * @param page the page number to retrieve. - */ @Suppress("DEPRECATION") - suspend fun getPopularManga(page: Int): MangasPage = fetchPopularManga(page).awaitSingle() + override suspend fun getPopularManga(page: Int): MangasPage = fetchPopularManga(page).awaitSingle() - /** - * Get a page with a list of manga. - * - * @since extensions-lib 1.5 - * @param page the page number to retrieve. - * @param query the search query. - * @param filters the list of filters to apply. - */ @Suppress("DEPRECATION") - suspend fun getSearchManga( + override suspend fun getLatestUpdates(page: Int): MangasPage = fetchLatestUpdates(page).awaitSingle() + + @Suppress("DEPRECATION") + override suspend fun getSearchManga( page: Int, query: String, filters: FilterList, ): MangasPage = fetchSearchManga(page, query, filters).awaitSingle() + @Suppress("DEPRECATION") + override suspend fun getMangaUpdate( + manga: SManga, + chapters: List, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate = + supervisorScope { + val asyncManga = if (fetchDetails) async { fetchMangaDetails(manga).awaitSingle() } else null + val asyncChapters = if (fetchChapters) async { fetchChapterList(manga).awaitSingle() } else null + SMangaUpdate(asyncManga?.await() ?: manga, asyncChapters?.await() ?: chapters) + } + + @Suppress("DEPRECATION") + override suspend fun getPageList(chapter: SChapter): List = fetchPageList(chapter).awaitSingle() + /** - * Get a page with a list of latest manga updates. + * Returns an observable containing a page with a list of manga. * - * @since extensions-lib 1.5 * @param page the page number to retrieve. */ - @Suppress("DEPRECATION") - suspend fun getLatestUpdates(page: Int): MangasPage = fetchLatestUpdates(page).awaitSingle() + @Deprecated("Use the suspend API instead", ReplaceWith("getPopularManga")) + fun fetchPopularManga(page: Int): Observable = throw UnsupportedOperationException() /** - * Returns the list of filters for the source. + * Returns an observable containing a page with a list of manga. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. */ - fun getFilterList(): FilterList - - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getPopularManga"), - ) - fun fetchPopularManga(page: Int): Observable = throw IllegalStateException("Not used") - - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getSearchManga"), - ) + @Deprecated("Use the suspend API instead", ReplaceWith("getSearchManga")) fun fetchSearchManga( page: Int, query: String, filters: FilterList, - ): Observable = throw IllegalStateException("Not used") + ): Observable = throw UnsupportedOperationException() - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getLatestUpdates"), - ) - fun fetchLatestUpdates(page: Int): Observable = throw IllegalStateException("Not used") + /** + * Returns an observable containing a page with a list of latest manga updates. + * + * @param page the page number to retrieve. + */ + @Deprecated("Use the suspend API instead", ReplaceWith("getLatestUpdates")) + fun fetchLatestUpdates(page: Int): Observable = throw UnsupportedOperationException() } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/PreferenceScreen.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/PreferenceScreen.kt new file mode 100644 index 000000000..0e6d5cdc2 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/PreferenceScreen.kt @@ -0,0 +1,4 @@ +package eu.kanade.tachiyomi.source + +@Suppress("unused") +typealias PreferenceScreen = androidx.preference.PreferenceScreen diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt index 76f2dfe80..e7e6ddefc 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt @@ -1,10 +1,12 @@ package eu.kanade.tachiyomi.source +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.SMangaUpdate import rx.Observable -import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle /** * A basic interface for creating a source. It could be an online source, a local source, etc. @@ -24,53 +26,86 @@ interface Source { get() = "" /** - * Get the updated details for a manga. - * - * @since extensions-lib 1.5 - * @param manga the manga to update. - * @return the updated manga. + * Whether the source has support for latest updates. */ - @Suppress("DEPRECATION") - suspend fun getMangaDetails(manga: SManga): SManga = fetchMangaDetails(manga).awaitSingle() + val supportsLatest: Boolean /** - * Get all the available chapters for a manga. - * - * @since extensions-lib 1.5 - * @param manga the manga to update. - * @return the chapters for the manga. + * Returns the list of filters for the source. */ - @Suppress("DEPRECATION") - suspend fun getChapterList(manga: SManga): List = fetchChapterList(manga).awaitSingle() + fun getFilterList(): FilterList = FilterList() + + /** + * Get a page with a list of manga. + * + * @since tachiyomix 1.6 + * @param page the page number to retrieve. + */ + suspend fun getPopularManga(page: Int): MangasPage + + /** + * Get a page with a list of latest manga updates. + * + * @since tachiyomix 1.6 + * @param page the page number to retrieve. + */ + suspend fun getLatestUpdates(page: Int): MangasPage + + /** + * Get a page with a list of manga. + * + * @since tachiyomix 1.6 + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + suspend fun getSearchManga( + page: Int, + query: String, + filters: FilterList, + ): MangasPage + + /** + * Fetches updated information for a manga. + * + * Depending on the provided flags or source availability, this may include + * updated manga metadata, available chapters, or both. + * + * If a value is not requested, the existing provided value can be returned as-is. + * The host app may apply any returned updates regardless of the flags, + * so care should be taken to only return accurate and intentional changes. + * + * @since tachiyomix 1.6 + * @param manga The manga to fetch updates for. + * @param chapters Existing chapters of the manga + * @param fetchDetails Whether to fetch updated manga details. + * @param fetchChapters Whether to fetch available chapters. + */ + suspend fun getMangaUpdate( + manga: SManga, + chapters: List, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate /** * Get the list of pages a chapter has. Pages should be returned * in the expected order; the index is ignored. * - * @since extensions-lib 1.5 + * @since tachiyomix 1.6 * @param chapter the chapter. * @return the pages for the chapter. */ - @Suppress("DEPRECATION") - suspend fun getPageList(chapter: SChapter): List = fetchPageList(chapter).awaitSingle() + suspend fun getPageList(chapter: SChapter): List - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getMangaDetails"), - ) - fun fetchMangaDetails(manga: SManga): Observable = throw IllegalStateException("Not used") + @Deprecated("Use the combined suspend API instead", ReplaceWith("getMangaUpdate")) + fun fetchMangaDetails(manga: SManga): Observable = throw UnsupportedOperationException() - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getChapterList"), - ) - fun fetchChapterList(manga: SManga): Observable> = throw IllegalStateException("Not used") + @Deprecated("Use the combined suspend API instead", ReplaceWith("getMangaUpdate")) + fun fetchChapterList(manga: SManga): Observable> = throw UnsupportedOperationException() - @Deprecated( - "Use the non-RxJava API instead", - ReplaceWith("getPageList"), - ) - fun fetchPageList(chapter: SChapter): Observable> = throw IllegalStateException("Not used") + @Deprecated("Use the suspend API instead", ReplaceWith("getPageList")) + fun fetchPageList(chapter: SChapter): Observable> = throw UnsupportedOperationException() } // fun Source.icon(): Drawable? = Injekt.get().getAppIconForSource(this) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt index 41035e2c1..0dc2b986a 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt @@ -23,12 +23,15 @@ import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.SMangaUpdate import eu.kanade.tachiyomi.util.chapter.ChapterRecognition import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder import eu.kanade.tachiyomi.util.storage.EpubFile import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.supervisorScope import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import kotlinx.serialization.json.decodeFromStream @@ -167,8 +170,20 @@ class LocalSource( return MangasPage(mangas.toList(), false) } + override suspend fun getMangaUpdate( + manga: SManga, + chapters: List, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate = + supervisorScope { + val asyncManga = if (fetchDetails) async { getMangaDetails(manga) } else null + val asyncChapters = if (fetchChapters) async { getChapterList(manga) } else null + SMangaUpdate(asyncManga?.await() ?: manga, asyncChapters?.await() ?: chapters) + } + // Manga details related - override suspend fun getMangaDetails(manga: SManga): SManga = + private suspend fun getMangaDetails(manga: SManga): SManga = withContext(Dispatchers.IO) { coverManager.find(manga.url)?.let { manga.thumbnail_url = it.absolutePath @@ -289,7 +304,7 @@ class LocalSource( } // Chapters - override suspend fun getChapterList(manga: SManga): List = + private suspend fun getChapterList(manga: SManga): List = fileSystem .getFilesInMangaDirectory(manga.url) // Only keep supported formats @@ -467,7 +482,8 @@ class LocalSource( it[versionName] = "1.2" it[versionCode] = 0 it[lang] = LANG - it[isNsfw] = false + it[extensionLib] = "1.2" + it[contentWarning] = 0 it[isInstalled] = true } @@ -476,7 +492,6 @@ class LocalSource( it[name] = NAME it[lang] = LANG it[extension] = extensionId - it[isNsfw] = false } } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/MangasPage.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/MangasPage.kt index cda439bf0..0f72131e1 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/MangasPage.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/MangasPage.kt @@ -1,6 +1,22 @@ package eu.kanade.tachiyomi.source.model -data class MangasPage( +class MangasPage( val mangas: List, val hasNextPage: Boolean, -) +) { + @Deprecated("MangasPage is now a regular class") + operator fun component1(): List = mangas + + @Deprecated("MangasPage is now a regular class") + operator fun component2(): Boolean = hasNextPage + + @Deprecated("MangasPage is now a regular class") + fun copy( + mangas: List = this.mangas, + hasNextPage: Boolean = this.hasNextPage, + ): MangasPage = + MangasPage( + mangas = mangas, + hasNextPage = hasNextPage, + ) +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt index 5bebace06..584fbdf61 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt @@ -27,12 +27,4 @@ open class Page( -1 } } - - companion object { - const val QUEUE = 0 - const val LOAD_PAGE = 1 - const val DOWNLOAD_IMAGE = 2 - const val READY = 3 - const val ERROR = 4 - } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapter.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapter.kt index 5ae70aa20..c6e5251ce 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapter.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapter.kt @@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.source.model +import kotlinx.serialization.json.JsonObject import java.io.Serializable interface SChapter : Serializable { @@ -9,12 +10,25 @@ interface SChapter : Serializable { var name: String - var date_upload: Long - var chapter_number: Float var scanlator: String? + var date_upload: Long + + /** + * Extra metadata associated with the chapter. + * + * The JSON object is not visible to users and intended for internal or source-specific + * purposes. Apps may define their own namespaced keys (e.g., `"mihon.*"`) for sources to populate. + * + * This allows apps to attach and ask for custom information without affecting the visible + * chapter data. + * + * @since tachiyomix 1.6 + */ + var memo: JsonObject + fun copyFrom(other: SChapter) { name = other.name url = other.url diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapterImpl.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapterImpl.kt index 7c96902d2..d583db909 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapterImpl.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SChapterImpl.kt @@ -2,14 +2,19 @@ package eu.kanade.tachiyomi.source.model +import kotlinx.serialization.json.JsonObject +import suwayomi.tachidesk.manga.impl.util.lang.EMPTY + class SChapterImpl : SChapter { override lateinit var url: String override lateinit var name: String - override var date_upload: Long = 0 - override var chapter_number: Float = -1f override var scanlator: String? = null + + override var date_upload: Long = 0 + + override var memo: JsonObject = JsonObject.EMPTY } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt index f50e430ba..6ba982fab 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt @@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.source.model +import kotlinx.serialization.json.JsonObject import java.io.Serializable interface SManga : Serializable { @@ -9,22 +10,58 @@ interface SManga : Serializable { var title: String + var thumbnail_url: String? + var artist: String? var author: String? + var status: Int + var description: String? var genre: String? - var status: Int - - var thumbnail_url: String? - var update_strategy: UpdateStrategy var initialized: Boolean + /** + * Extra metadata associated with the manga. + * + * The JSON object is not visible to users and intended for internal or source-specific + * purposes. Apps may define their own namespaced keys (e.g., `"mihon.*"`) for sources to populate. + * + * This allows apps to attach and ask for custom information without affecting the visible + * manga data. + * + * @since tachiyomix 1.6 + */ + var memo: JsonObject + + fun getGenres(): List? { + if (genre.isNullOrBlank()) return null + return genre + ?.split(", ") + ?.map { it.trim() } + ?.filterNot { it.isBlank() } + ?.distinct() + } + + fun copy() = + create().also { + it.url = url + it.title = title + it.artist = artist + it.author = author + it.description = description + it.genre = genre + it.status = status + it.thumbnail_url = thumbnail_url + it.update_strategy = update_strategy + it.initialized = initialized + } + companion object { const val UNKNOWN = 0 const val ONGOING = 1 @@ -37,30 +74,3 @@ interface SManga : Serializable { fun create(): SManga = SMangaImpl() } } - -// fun SManga.toMangaInfo(): MangaInfo { -// return MangaInfo( -// key = this.url, -// title = this.title, -// artist = this.artist ?: "", -// author = this.author ?: "", -// description = this.description ?: "", -// genres = this.genre?.split(", ") ?: emptyList(), -// status = this.status, -// cover = this.thumbnail_url ?: "" -// ) -// } -// -// fun MangaInfo.toSManga(): SManga { -// val mangaInfo = this -// return SManga.create().apply { -// url = mangaInfo.key -// title = mangaInfo.title -// artist = mangaInfo.artist -// author = mangaInfo.author -// description = mangaInfo.description -// genre = mangaInfo.genres.joinToString(", ") -// status = mangaInfo.status -// thumbnail_url = mangaInfo.cover -// } -// } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt index fa696dd19..514a333b2 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt @@ -2,24 +2,29 @@ package eu.kanade.tachiyomi.source.model +import kotlinx.serialization.json.JsonObject +import suwayomi.tachidesk.manga.impl.util.lang.EMPTY + class SMangaImpl : SManga { override lateinit var url: String override lateinit var title: String + override var thumbnail_url: String? = null + override var artist: String? = null override var author: String? = null + override var status: Int = 0 + override var description: String? = null override var genre: String? = null - override var status: Int = 0 - - override var thumbnail_url: String? = null - override var update_strategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE override var initialized: Boolean = false + + override var memo: JsonObject = JsonObject.EMPTY } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaUpdate.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaUpdate.kt new file mode 100644 index 000000000..19a692f87 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaUpdate.kt @@ -0,0 +1,7 @@ +package eu.kanade.tachiyomi.source.model + +@Suppress("UNUSED") +class SMangaUpdate( + val manga: SManga, + val chapters: List, +) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt index 2ebd485c8..91b5f5e29 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt @@ -1,6 +1,22 @@ package eu.kanade.tachiyomi.source.model +/** + * Define the update strategy for a single [SManga]. + * The strategy used will only take effect on the library update. + * + * @since extensions-lib 1.4 + */ enum class UpdateStrategy { + /** + * Series marked as always update will be included in the library + * update if they aren't excluded by additional restrictions. + */ ALWAYS_UPDATE, + + /** + * Series marked as only fetch once will be automatically skipped + * during library updates. Useful for cases where the series is previously + * known to be finished and have only a single chapter, for example. + */ ONLY_FETCH_ONCE, } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt index 966b67c4f..1a5719718 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt @@ -25,7 +25,6 @@ import java.security.MessageDigest /** * A simple implementation for sources from a website. */ -@Suppress("unused") abstract class HttpSource : CatalogueSource { /** * Network service. @@ -37,11 +36,24 @@ abstract class HttpSource : CatalogueSource { */ abstract val baseUrl: String + /** + * Returns the base (home) URL of the website as a string. + * + * This is typically the root address that serves as the main entry point + * to the site's content, such as "https://mihon.tech". + * + * This method is used in the browse screen to determine the URL + * opened when tapping "Open in WebView". + * + * @return The website’s home page URL. Defaults to [baseUrl]. + */ + open fun getHomeUrl(): String = baseUrl + /** * Version id used to generate the source id. If the site completely changes and urls are * incompatible, you may increase this value and it'll be considered as a new source. */ - open val versionId = 1 + open val versionId: Int = 1 /** * ID of the source. By default it uses a generated id using the first 16 characters (64 bits) @@ -53,7 +65,7 @@ abstract class HttpSource : CatalogueSource { * * Note: the generated ID sets the sign bit to `0`. */ - override val id by lazy { generateId() } + override val id: Long by lazy { generateId(name, lang, versionId) } /** * Headers used for requests. @@ -63,10 +75,7 @@ abstract class HttpSource : CatalogueSource { /** * Default network client for doing requests. */ - open val client: OkHttpClient - get() = network.client - - private fun generateId(): Long = generateId("${name.lowercase()}/$lang/$versionId") + open val client: OkHttpClient get() = network.client /** * Generates a unique ID for the source based on the provided [name], [lang] and @@ -91,10 +100,6 @@ abstract class HttpSource : CatalogueSource { versionId: Int, ): Long { val key = "${name.lowercase()}/$lang/$versionId" - return generateId(key) - } - - private fun generateId(key: String): Long { val bytes = MessageDigest.getInstance("MD5").digest(key.toByteArray()) return (0..7).map { bytes[it].toLong() and 0xff shl 8 * (7 - it) }.reduce(Long::or) and Long.MAX_VALUE } @@ -102,7 +107,7 @@ abstract class HttpSource : CatalogueSource { /** * Headers builder for requests. Implementations can override this method for custom headers. */ - protected open fun headersBuilder() = + protected open fun headersBuilder(): Headers.Builder = Headers.Builder().apply { add("User-Agent", network.defaultUserAgentProvider()) } @@ -110,7 +115,7 @@ abstract class HttpSource : CatalogueSource { /** * Visible name of the source. */ - override fun toString() = "$name (${lang.uppercase()})" + override fun toString(): String = "$name (${lang.uppercase()})" /** * Returns an observable containing a page with a list of manga. Normally it's not needed to @@ -118,7 +123,8 @@ abstract class HttpSource : CatalogueSource { * * @param page the page number to retrieve. */ - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPopularManga")) + @Suppress("DEPRECATION") + @Deprecated("Use the suspend API instead", ReplaceWith("getPopularManga")) override fun fetchPopularManga(page: Int): Observable = client .newCall(popularMangaRequest(page)) @@ -132,14 +138,24 @@ abstract class HttpSource : CatalogueSource { * * @param page the page number to retrieve. */ - protected abstract fun popularMangaRequest(page: Int): Request + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun popularMangaRequest(page: Int): Request = throw UnsupportedOperationException() /** * Parses the response from the site and returns a [MangasPage] object. * * @param response the response from the site. */ - protected abstract fun popularMangaParse(response: Response): MangasPage + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun popularMangaParse(response: Response): MangasPage = throw UnsupportedOperationException() /** * Returns an observable containing a page with a list of manga. Normally it's not needed to @@ -149,22 +165,17 @@ abstract class HttpSource : CatalogueSource { * @param query the search query. * @param filters the list of filters to apply. */ - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getSearchManga")) + @Suppress("DEPRECATION") + @Deprecated("Use the suspend API instead", ReplaceWith("getSearchManga")) override fun fetchSearchManga( page: Int, query: String, filters: FilterList, ): Observable = - Observable - .defer { - try { - client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess() - } catch (e: NoClassDefFoundError) { - // RxJava doesn't handle Errors, which tends to happen during global searches - // if an old extension using non-existent classes is still around - throw RuntimeException(e) - } - }.map { response -> + client + .newCall(searchMangaRequest(page, query, filters)) + .asObservableSuccess() + .map { response -> searchMangaParse(response) } @@ -175,25 +186,36 @@ abstract class HttpSource : CatalogueSource { * @param query the search query. * @param filters the list of filters to apply. */ - protected abstract fun searchMangaRequest( + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun searchMangaRequest( page: Int, query: String, filters: FilterList, - ): Request + ): Request = throw UnsupportedOperationException() /** * Parses the response from the site and returns a [MangasPage] object. * * @param response the response from the site. */ - protected abstract fun searchMangaParse(response: Response): MangasPage + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun searchMangaParse(response: Response): MangasPage = throw UnsupportedOperationException() /** * Returns an observable containing a page with a list of latest manga updates. * * @param page the page number to retrieve. */ - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getLatestUpdates")) + @Suppress("DEPRECATION") + @Deprecated("Use the suspend API instead", ReplaceWith("getLatestUpdates")) override fun fetchLatestUpdates(page: Int): Observable = client .newCall(latestUpdatesRequest(page)) @@ -207,26 +229,33 @@ abstract class HttpSource : CatalogueSource { * * @param page the page number to retrieve. */ - protected abstract fun latestUpdatesRequest(page: Int): Request + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun latestUpdatesRequest(page: Int): Request = throw UnsupportedOperationException() /** * Parses the response from the site and returns a [MangasPage] object. * * @param response the response from the site. */ - protected abstract fun latestUpdatesParse(response: Response): MangasPage + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun latestUpdatesParse(response: Response): MangasPage = throw UnsupportedOperationException() /** - * Get the updated details for a manga. - * Normally it's not needed to override this method. + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. * - * @param manga the manga to update. - * @return the updated manga. + * @param manga the manga to be updated. */ @Suppress("DEPRECATION") - override suspend fun getMangaDetails(manga: SManga): SManga = fetchMangaDetails(manga).awaitSingle() - - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getMangaDetails")) + @Deprecated("Use the combined suspend API instead", replaceWith = ReplaceWith("getMangaUpdate")) override fun fetchMangaDetails(manga: SManga): Observable = client .newCall(mangaDetailsRequest(manga)) @@ -241,6 +270,11 @@ abstract class HttpSource : CatalogueSource { * * @param manga the manga to be updated. */ + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) open fun mangaDetailsRequest(manga: SManga): Request = GET(baseUrl + manga.url, headers) /** @@ -248,37 +282,28 @@ abstract class HttpSource : CatalogueSource { * * @param response the response from the site. */ - protected abstract fun mangaDetailsParse(response: Response): SManga + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun mangaDetailsParse(response: Response): SManga = throw UnsupportedOperationException() /** - * Get all the available chapters for a manga. - * Normally it's not needed to override this method. + * Returns an observable with the updated chapter list for a manga. Normally it's not needed to + * override this method. * - * @param manga the manga to update. - * @return the chapters for the manga. - * @throws LicensedMangaChaptersException if a manga is licensed and therefore no chapters are available. + * @param manga the manga to look for chapters. */ @Suppress("DEPRECATION") - override suspend fun getChapterList(manga: SManga): List { - if (manga.status == SManga.LICENSED) { - throw LicensedMangaChaptersException() - } - - return fetchChapterList(manga).awaitSingle() - } - - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getChapterList")) + @Deprecated("Use the combined suspend API instead", replaceWith = ReplaceWith("getMangaUpdate")) override fun fetchChapterList(manga: SManga): Observable> = - if (manga.status != SManga.LICENSED) { - client - .newCall(chapterListRequest(manga)) - .asObservableSuccess() - .map { response -> - chapterListParse(response) - } - } else { - Observable.error(LicensedMangaChaptersException()) - } + client + .newCall(chapterListRequest(manga)) + .asObservableSuccess() + .map { response -> + chapterListParse(response) + } /** * Returns the request for updating the chapter list. Override only if it's needed to override @@ -286,6 +311,11 @@ abstract class HttpSource : CatalogueSource { * * @param manga the manga to look for chapters. */ + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) protected open fun chapterListRequest(manga: SManga): Request = GET(baseUrl + manga.url, headers) /** @@ -293,19 +323,20 @@ abstract class HttpSource : CatalogueSource { * * @param response the response from the site. */ - protected abstract fun chapterListParse(response: Response): List + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun chapterListParse(response: Response): List = throw UnsupportedOperationException() /** - * Get the list of pages a chapter has. Pages should be returned - * in the expected order; the index is ignored. + * Returns an observable with the page list for a chapter. * - * @param chapter the chapter. - * @return the pages for the chapter. + * @param chapter the chapter whose page list has to be fetched. */ @Suppress("DEPRECATION") - override suspend fun getPageList(chapter: SChapter): List = fetchPageList(chapter).awaitSingle() - - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPageList")) + @Deprecated("Use the suspend API instead", ReplaceWith("getPageList")) override fun fetchPageList(chapter: SChapter): Observable> = client .newCall(pageListRequest(chapter)) @@ -320,6 +351,11 @@ abstract class HttpSource : CatalogueSource { * * @param chapter the chapter whose page list has to be fetched. */ + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) protected open fun pageListRequest(chapter: SChapter): Request = GET(baseUrl + chapter.url, headers) /** @@ -327,31 +363,47 @@ abstract class HttpSource : CatalogueSource { * * @param response the response from the site. */ - protected abstract fun pageListParse(response: Response): List + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun pageListParse(response: Response): List = throw UnsupportedOperationException() /** * Returns an observable with the page containing the source url of the image. If there's any * error, it will return null instead of throwing an exception. * - * @since extensions-lib 1.5 * @param page the page whose source image has to be fetched. */ @Suppress("DEPRECATION") - open suspend fun getImageUrl(page: Page): String = fetchImageUrl(page).awaitSingle() - - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getImageUrl")) + @Deprecated("Use the suspend API instead", ReplaceWith("getImageUrl")) open fun fetchImageUrl(page: Page): Observable = client .newCall(imageUrlRequest(page)) .asObservableSuccess() .map { imageUrlParse(it) } + /** + * Returns the image url for the provided [page]. The function is only called if [Page.imageUrl] is null. + * + * @since tachiyomix 1.6 + * @param page the page whose source image has to be fetched. + */ + @Suppress("DEPRECATION") + open suspend fun getImageUrl(page: Page): String = fetchImageUrl(page).awaitSingle() + /** * Returns the request for getting the url to the source image. Override only if it's needed to * override the url, send different headers or request method like POST. * * @param page the chapter whose page list has to be fetched */ + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) protected open fun imageUrlRequest(page: Page): Request = GET(page.url, headers) /** @@ -359,16 +411,14 @@ abstract class HttpSource : CatalogueSource { * * @param response the response from the site. */ - protected abstract fun imageUrlParse(response: Response): String + @Deprecated( + message = + "The helper functions are inherently limiting and hides the underlying implementation. " + + "Source developers should make their own implementation according to their needs.", + ) + protected open fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - /** - * Returns the response of the source image. - * Typically does not need to be overridden. - * - * @since extensions-lib 1.5 - * @param page the page whose source image has to be downloaded. - */ - open suspend fun getImage(page: Page): Response = + suspend fun getImage(page: Page): Response = client .newCachelessCallWithProgress(imageRequest(page), page) .awaitSuccess() @@ -387,6 +437,7 @@ abstract class HttpSource : CatalogueSource { * * @param url the full url to the chapter. */ + @Suppress("Unused") fun SChapter.setUrlWithoutDomain(url: String) { this.url = getUrlWithoutDomain(url) } @@ -397,6 +448,7 @@ abstract class HttpSource : CatalogueSource { * * @param url the full url to the manga. */ + @Suppress("Unused") fun SManga.setUrlWithoutDomain(url: String) { this.url = getUrlWithoutDomain(url) } @@ -417,7 +469,7 @@ abstract class HttpSource : CatalogueSource { out += "#" + uri.fragment } out - } catch (e: URISyntaxException) { + } catch (_: URISyntaxException) { orig } @@ -428,6 +480,7 @@ abstract class HttpSource : CatalogueSource { * @param manga the manga * @return url of the manga */ + @Suppress("DEPRECATION") open fun getMangaUrl(manga: SManga): String = mangaDetailsRequest(manga).url.toString() /** @@ -437,6 +490,7 @@ abstract class HttpSource : CatalogueSource { * @param chapter the chapter * @return url of the chapter */ + @Suppress("DEPRECATION") open fun getChapterUrl(chapter: SChapter): String = pageListRequest(chapter).url.toString() /** @@ -446,15 +500,9 @@ abstract class HttpSource : CatalogueSource { * @param chapter the chapter to be added. * @param manga the manga of the chapter. */ + @Deprecated("All modifications should be done when constructing the chapter") open fun prepareNewChapter( chapter: SChapter, manga: SManga, ) {} - - /** - * Returns the list of filters for the source. - */ - override fun getFilterList() = FilterList() } - -class LicensedMangaChaptersException : Exception("Licensed - No chapters to show") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt index 66db603b9..ec5648463 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt @@ -12,12 +12,20 @@ import org.jsoup.nodes.Element /** * A simple implementation for sources from a website using Jsoup, an HTML parser. */ +@Deprecated( + message = + "In most cases sources only require a subset of the methods from this class. " + + "Source developers should make their own implementation according to their needs.", +) abstract class ParsedHttpSource : HttpSource() { /** * Parses the response from the site and returns a [MangasPage] object. * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun popularMangaParse(response: Response): MangasPage { val document = response.asJsoup() @@ -58,6 +66,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun searchMangaParse(response: Response): MangasPage { val document = response.asJsoup() @@ -98,6 +109,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun latestUpdatesParse(response: Response): MangasPage { val document = response.asJsoup() @@ -138,6 +152,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) /** @@ -152,6 +169,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun chapterListParse(response: Response): List { val document = response.asJsoup() return document.select(chapterListSelector()).map { chapterFromElement(it) } @@ -174,6 +194,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) /** @@ -188,6 +211,9 @@ abstract class ParsedHttpSource : HttpSource() { * * @param response the response from the site. */ + @Deprecated( + "The helper functions are inherently limiting and hides the underlying implementation. Source developers should make their own implementation according to their needs.", + ) override fun imageUrlParse(response: Response): String = imageUrlParse(response.asJsoup()) /** diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ResolvableSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ResolvableSource.kt index 0e9431692..59f732cf8 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ResolvableSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/ResolvableSource.kt @@ -1,26 +1,44 @@ package eu.kanade.tachiyomi.source.online import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga /** - * A source that may handle opening an SManga for a given URI. + * A source that may handle opening an SManga or SChapter for a given URI. * * @since extensions-lib 1.5 */ -@Suppress("unused") interface ResolvableSource : Source { /** - * Whether this source may potentially handle the given URI. + * Returns what the given URI may open. + * Returns [UriType.Unknown] if the source is not able to resolve the URI. * * @since extensions-lib 1.5 */ - fun canResolveUri(uri: String): Boolean + fun getUriType(uri: String): UriType /** - * Called if canHandleUri is true. Returns the corresponding SManga, if possible. + * Called if [getUriType] is [UriType.Manga]. + * Returns the corresponding SManga, if possible. * * @since extensions-lib 1.5 */ suspend fun getManga(uri: String): SManga? + + /** + * Called if [getUriType] is [UriType.Chapter]. + * Returns the corresponding SChapter, if possible. + * + * @since extensions-lib 1.5 + */ + suspend fun getChapter(uri: String): SChapter? +} + +sealed interface UriType { + data object Manga : UriType + + data object Chapter : UriType + + data object Unknown : UriType } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt new file mode 100644 index 000000000..77f6181e3 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt @@ -0,0 +1,57 @@ +package suwayomi.tachidesk.graphql.dataLoaders + +import com.expediagroup.graphql.dataloader.KotlinDataLoader +import graphql.GraphQLContext +import org.dataloader.DataLoader +import org.dataloader.DataLoaderFactory +import org.jetbrains.exposed.v1.core.Slf4jSqlDebugLogger +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.types.ExtensionNodeList +import suwayomi.tachidesk.graphql.types.ExtensionNodeList.Companion.toNodeList +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.graphql.types.ExtensionType +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.manga.model.table.ExtensionTable +import suwayomi.tachidesk.server.JavalinSetup.future + +class ExtensionStoreDataLoader : KotlinDataLoader { + override val dataLoaderName = "ExtensionStoreDataLoader" + + override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = + DataLoaderFactory.newDataLoader { ids -> + future { + transaction { + addLogger(Slf4jSqlDebugLogger) + val extensionStoreByIndexUrl = + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl inList ids } + .map { ExtensionStoreType(it) } + .associateBy { it.indexUrl } + ids.map { extensionStoreByIndexUrl[it] } + } + } + } +} + +class ExtensionsForExtensionStore : KotlinDataLoader { + override val dataLoaderName = "ExtensionsForExtensionStore" + + override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = + DataLoaderFactory.newDataLoader { ids -> + future { + transaction { + addLogger(Slf4jSqlDebugLogger) + val extensionByIndexUrl = + ExtensionTable + .selectAll() + .where { ExtensionTable.storeIndexUrl inList ids } + .map { ExtensionType(it) } + .groupBy { it.storeIndexUrl } + ids.map { (extensionByIndexUrl[it] ?: emptyList()).toNodeList() } + } + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt index 0c91186ec..78e16f244 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt @@ -2,6 +2,7 @@ package suwayomi.tachidesk.graphql.mutations +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import org.jetbrains.exposed.v1.core.LikePattern @@ -25,6 +26,7 @@ import suwayomi.tachidesk.graphql.types.ChapterType import suwayomi.tachidesk.graphql.types.MetaInput import suwayomi.tachidesk.graphql.types.SyncConflictInfoType import suwayomi.tachidesk.manga.impl.Chapter +import suwayomi.tachidesk.manga.impl.Manga import suwayomi.tachidesk.manga.impl.chapter.getChapterDownloadReadyById import suwayomi.tachidesk.manga.impl.sync.KoreaderSyncService import suwayomi.tachidesk.manga.model.table.ChapterMetaTable @@ -167,11 +169,12 @@ class ChapterMutation { ) @RequireAuth + @GraphQLDeprecated("Deprecated in Tachiyomix 1.6", ReplaceWith("fetchMangaAndChapters")) fun fetchChapters(input: FetchChaptersInput): CompletableFuture { val (clientMutationId, mangaId) = input return future { - Chapter.fetchChapterList(mangaId) + Manga.updateMangaAndChapters(mangaId, updateManga = false) val chapters = transaction { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt index 663f32c5c..4246dd468 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt @@ -10,9 +10,11 @@ import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.ExtensionStoreType import suwayomi.tachidesk.graphql.types.ExtensionType import suwayomi.tachidesk.manga.impl.extension.Extension import suwayomi.tachidesk.manga.impl.extension.ExtensionsList +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable import suwayomi.tachidesk.manga.model.table.ExtensionTable import suwayomi.tachidesk.server.JavalinSetup.future import java.util.concurrent.CompletableFuture @@ -129,6 +131,7 @@ class ExtensionMutation { data class FetchExtensionsPayload( val clientMutationId: String?, val extensions: List, + val extensionStores: List, ) @RequireAuth @@ -146,9 +149,17 @@ class ExtensionMutation { .map { ExtensionType(it) } } + val extensionStores = + transaction { + ExtensionStoreTable + .selectAll() + .map { ExtensionStoreType(it) } + } + FetchExtensionsPayload( clientMutationId = clientMutationId, extensions = extensions, + extensionStores = extensionStores, ) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt new file mode 100644 index 000000000..1fa9b190d --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt @@ -0,0 +1,104 @@ +package suwayomi.tachidesk.graphql.mutations + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.manga.impl.extension.ExtensionStoreService +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.server.JavalinSetup.future +import java.util.concurrent.CompletableFuture + +class ExtensionStoreMutation { + data class AddExtensionStoreInput( + val clientMutationId: String? = null, + val indexUrl: String, + ) + + data class AddExtensionStorePayload( + val clientMutationId: String?, + val extensionStore: ExtensionStoreType, + ) + + @RequireAuth + fun addExtensionStore(input: AddExtensionStoreInput): CompletableFuture { + val (clientMutationId, indexUrl) = input + return future { + val store = ExtensionStoreService.fetch(indexUrl) + + ExtensionStoreService.upsert(store) + ExtensionStoreService.syncDbToPrefs() + val row = + transaction { + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq store.indexUrl } + .first() + } + + AddExtensionStorePayload( + clientMutationId = clientMutationId, + extensionStore = ExtensionStoreType(row), + ) + } + } + + data class RemoveExtensionStoreInput( + val clientMutationId: String? = null, + val indexUrl: String, + ) + + data class RemoveExtensionStorePayload( + val clientMutationId: String?, + val extensionStore: ExtensionStoreType?, + ) + + @RequireAuth + fun removeExtensionStore(input: RemoveExtensionStoreInput): CompletableFuture { + val (clientMutationId, indexUrl) = input + return future { + val store = + transaction { + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq indexUrl } + .firstOrNull() + ?.let { ExtensionStoreType(it) } + } + + store?.let { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl eq indexUrl } + } + } + + ExtensionStoreService.syncDbToPrefs() + + RemoveExtensionStorePayload( + clientMutationId = clientMutationId, + extensionStore = + store?.let { + ExtensionStoreType( + name = it.name, + badgeLabel = it.badgeLabel, + signingKey = it.signingKey, + contactWebsite = it.contactWebsite, + contactDiscord = it.contactDiscord, + indexUrl = it.indexUrl, + isLegacy = it.isLegacy, + extensionListUrl = it.extensionListUrl, + ) + }, + ) + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt index 975f55678..00c2a6224 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt @@ -2,6 +2,7 @@ package suwayomi.tachidesk.graphql.mutations +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated import org.jetbrains.exposed.v1.core.LikePattern import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.and @@ -14,12 +15,14 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.ChapterType import suwayomi.tachidesk.graphql.types.MangaMetaType import suwayomi.tachidesk.graphql.types.MangaType import suwayomi.tachidesk.graphql.types.MetaInput import suwayomi.tachidesk.manga.impl.Library import suwayomi.tachidesk.manga.impl.Manga import suwayomi.tachidesk.manga.impl.update.IUpdater +import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.manga.model.table.MangaMetaTable import suwayomi.tachidesk.manga.model.table.MangaTable import suwayomi.tachidesk.manga.model.table.toDataClass @@ -146,11 +149,12 @@ class MangaMutation { ) @RequireAuth + @GraphQLDeprecated("Deprecated in Tachiyomix 1.6", ReplaceWith("fetchMangaAndChapters")) fun fetchManga(input: FetchMangaInput): CompletableFuture { val (clientMutationId, id) = input return future { - Manga.fetchManga(id) + Manga.updateMangaAndChapters(id, updateChapters = false) val manga = transaction { @@ -163,6 +167,49 @@ class MangaMutation { } } + data class FetchMangaAndChaptersInput( + val clientMutationId: String? = null, + val id: Int, + val fetchManga: Boolean, + val fetchChapters: Boolean, + ) + + data class FetchMangaAndChaptersPayload( + val clientMutationId: String?, + val manga: MangaType, + val chapters: List, + ) + + @RequireAuth + fun fetchMangaAndChapters(input: FetchMangaAndChaptersInput): CompletableFuture { + val (clientMutationId, id, fetchManga, fetchChapters) = input + + return future { + Manga.updateMangaAndChapters( + mangaId = id, + updateManga = fetchManga, + updateChapters = fetchChapters, + ) + + val (manga, chapters) = + transaction { + Pair( + MangaTable.selectAll().where { MangaTable.id eq id }.first(), + ChapterTable + .selectAll() + .where { ChapterTable.manga eq id } + .orderBy(ChapterTable.sourceOrder) + .map { ChapterType(it) }, + ) + } + FetchMangaAndChaptersPayload( + clientMutationId = clientMutationId, + manga = MangaType(manga), + chapters = chapters, + ) + } + } + data class SetMangaMetaInput( val clientMutationId: String? = null, val meta: MangaMetaType, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt index c2b1a8da6..be0ab67f2 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt @@ -21,12 +21,15 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.graphql.directives.RequireAuth import suwayomi.tachidesk.graphql.queries.filter.BooleanFilter +import suwayomi.tachidesk.graphql.queries.filter.ContentWarningFilter import suwayomi.tachidesk.graphql.queries.filter.Filter import suwayomi.tachidesk.graphql.queries.filter.HasGetOp import suwayomi.tachidesk.graphql.queries.filter.IntFilter +import suwayomi.tachidesk.graphql.queries.filter.LongFilter import suwayomi.tachidesk.graphql.queries.filter.OpAnd import suwayomi.tachidesk.graphql.queries.filter.StringFilter import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompare +import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareEnum import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareString import suwayomi.tachidesk.graphql.queries.filter.applyOps import suwayomi.tachidesk.graphql.server.primitives.Cursor @@ -40,6 +43,7 @@ import suwayomi.tachidesk.graphql.server.primitives.lessNotUnique import suwayomi.tachidesk.graphql.server.primitives.maybeSwap import suwayomi.tachidesk.graphql.types.ExtensionNodeList import suwayomi.tachidesk.graphql.types.ExtensionType +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.table.ExtensionTable import java.util.concurrent.CompletableFuture @@ -55,21 +59,23 @@ class ExtensionQuery { ) : OrderBy { PKG_NAME(ExtensionTable.pkgName), NAME(ExtensionTable.name), - APK_NAME(ExtensionTable.apkName), + + @GraphQLDeprecated("") + APK_NAME(ExtensionTable.pkgName), ; override fun greater(cursor: Cursor): Op = when (this) { PKG_NAME -> ExtensionTable.pkgName greater cursor.value NAME -> greaterNotUnique(ExtensionTable.name, ExtensionTable.pkgName, cursor, String::toString) - APK_NAME -> greaterNotUnique(ExtensionTable.apkName, ExtensionTable.pkgName, cursor, String::toString) + APK_NAME -> ExtensionTable.pkgName greater cursor.value } override fun less(cursor: Cursor): Op = when (this) { PKG_NAME -> ExtensionTable.pkgName less cursor.value NAME -> lessNotUnique(ExtensionTable.name, ExtensionTable.pkgName, cursor, String::toString) - APK_NAME -> lessNotUnique(ExtensionTable.apkName, ExtensionTable.pkgName, cursor, String::toString) + APK_NAME -> ExtensionTable.pkgName less cursor.value } override fun asCursor(type: ExtensionType): Cursor { @@ -89,29 +95,44 @@ class ExtensionQuery { ) : Order data class ExtensionCondition( + val storeIndexUrl: String? = null, + @GraphQLDeprecated("", ReplaceWith("storeIndexUrl")) val repo: String? = null, val apkName: String? = null, val iconUrl: String? = null, val name: String? = null, val pkgName: String? = null, + val apkUrl: String? = null, + val extensionLib: String? = null, val versionName: String? = null, val versionCode: Int? = null, + val versionCodeLong: Long? = null, val lang: String? = null, + @GraphQLDeprecated("", ReplaceWith("contentWarning")) val isNsfw: Boolean? = null, + val contentWarning: ContentWarning? = null, val isInstalled: Boolean? = null, val hasUpdate: Boolean? = null, val isObsolete: Boolean? = null, ) : HasGetOp { override fun getOp(): Op? { val opAnd = OpAnd() - opAnd.eq(repo, ExtensionTable.repo) + opAnd.eq(storeIndexUrl, ExtensionTable.storeIndexUrl) + opAnd.eq(repo, ExtensionTable.storeIndexUrl) opAnd.eq(apkName, ExtensionTable.apkName) opAnd.eq(iconUrl, ExtensionTable.iconUrl) + opAnd.eq(apkUrl, ExtensionTable.apkUrl) opAnd.eq(name, ExtensionTable.name) + opAnd.eq(extensionLib, ExtensionTable.extensionLib) opAnd.eq(versionName, ExtensionTable.versionName) - opAnd.eq(versionCode, ExtensionTable.versionCode) + opAnd.eq(versionCode?.toLong(), ExtensionTable.versionCode) + opAnd.eq(versionCodeLong, ExtensionTable.versionCode) opAnd.eq(lang, ExtensionTable.lang) - opAnd.eq(isNsfw, ExtensionTable.isNsfw) + opAnd.eq( + isNsfw?.let { if (it) ContentWarning.MIXED.ordinal else ContentWarning.SAFE.ordinal }, + ExtensionTable.contentWarning, + ) + opAnd.eq(contentWarning?.ordinal, ExtensionTable.contentWarning) opAnd.eq(isInstalled, ExtensionTable.isInstalled) opAnd.eq(hasUpdate, ExtensionTable.hasUpdate) opAnd.eq(isObsolete, ExtensionTable.isObsolete) @@ -121,15 +142,23 @@ class ExtensionQuery { } data class ExtensionFilter( + val storeIndexUrl: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("storeIndexUrl")) val repo: StringFilter? = null, val apkName: StringFilter? = null, val iconUrl: StringFilter? = null, val name: StringFilter? = null, val pkgName: StringFilter? = null, + val apkUrl: StringFilter? = null, val versionName: StringFilter? = null, + val extensionLib: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("versionCodeLong")) val versionCode: IntFilter? = null, + val versionCodeLong: LongFilter? = null, val lang: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("contentWarning")) val isNsfw: BooleanFilter? = null, + val contentWarning: ContentWarningFilter? = null, val isInstalled: BooleanFilter? = null, val hasUpdate: BooleanFilter? = null, val isObsolete: BooleanFilter? = null, @@ -139,15 +168,18 @@ class ExtensionQuery { ) : Filter { override fun getOpList(): List> = listOfNotNull( - andFilterWithCompareString(ExtensionTable.repo, repo), + andFilterWithCompareString(ExtensionTable.storeIndexUrl, storeIndexUrl), + andFilterWithCompareString(ExtensionTable.storeIndexUrl, repo), andFilterWithCompareString(ExtensionTable.apkName, apkName), andFilterWithCompareString(ExtensionTable.iconUrl, iconUrl), andFilterWithCompareString(ExtensionTable.name, name), andFilterWithCompareString(ExtensionTable.pkgName, pkgName), + andFilterWithCompareString(ExtensionTable.apkUrl, apkUrl), + andFilterWithCompareString(ExtensionTable.extensionLib, extensionLib), andFilterWithCompareString(ExtensionTable.versionName, versionName), - andFilterWithCompare(ExtensionTable.versionCode, versionCode), + andFilterWithCompare(ExtensionTable.versionCode, versionCodeLong), andFilterWithCompareString(ExtensionTable.lang, lang), - andFilterWithCompare(ExtensionTable.isNsfw, isNsfw), + andFilterWithCompareEnum(ExtensionTable.contentWarning, contentWarning), andFilterWithCompare(ExtensionTable.isInstalled, isInstalled), andFilterWithCompare(ExtensionTable.hasUpdate, hasUpdate), andFilterWithCompare(ExtensionTable.isObsolete, isObsolete), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt new file mode 100644 index 000000000..3e8468ec0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt @@ -0,0 +1,190 @@ +package suwayomi.tachidesk.graphql.queries + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import com.expediagroup.graphql.server.extensions.getValueFromDataLoader +import graphql.schema.DataFetchingEnvironment +import org.jetbrains.exposed.v1.core.Column +import org.jetbrains.exposed.v1.core.Op +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.queries.filter.Filter +import suwayomi.tachidesk.graphql.queries.filter.HasGetOp +import suwayomi.tachidesk.graphql.queries.filter.OpAnd +import suwayomi.tachidesk.graphql.queries.filter.StringFilter +import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareString +import suwayomi.tachidesk.graphql.queries.filter.applyOps +import suwayomi.tachidesk.graphql.server.primitives.Cursor +import suwayomi.tachidesk.graphql.server.primitives.Order +import suwayomi.tachidesk.graphql.server.primitives.OrderBy +import suwayomi.tachidesk.graphql.server.primitives.PageInfo +import suwayomi.tachidesk.graphql.server.primitives.QueryResults +import suwayomi.tachidesk.graphql.server.primitives.applyBeforeAfter +import suwayomi.tachidesk.graphql.server.primitives.greaterNotUnique +import suwayomi.tachidesk.graphql.server.primitives.lessNotUnique +import suwayomi.tachidesk.graphql.server.primitives.maybeSwap +import suwayomi.tachidesk.graphql.types.ExtensionStoreNodeList +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import java.util.concurrent.CompletableFuture + +class ExtensionStoreQuery { + @RequireAuth + fun extensionStore( + dataFetchingEnvironment: DataFetchingEnvironment, + indexUrl: String, + ): CompletableFuture = dataFetchingEnvironment.getValueFromDataLoader("ExtensionStoreDataLoader", indexUrl) + + enum class ExtensionStoreOrderBy( + override val column: Column<*>, + ) : OrderBy { + NAME(ExtensionStoreTable.name), + INDEX_URL(ExtensionStoreTable.indexUrl), + ; + + override fun greater(cursor: Cursor): Op = + when (this) { + NAME -> greaterNotUnique(ExtensionStoreTable.name, ExtensionStoreTable.id, cursor, String::toString) + INDEX_URL -> greaterNotUnique(ExtensionStoreTable.indexUrl, ExtensionStoreTable.id, cursor, String::toString) + } + + override fun less(cursor: Cursor): Op = + when (this) { + NAME -> lessNotUnique(ExtensionStoreTable.name, ExtensionStoreTable.id, cursor, String::toString) + INDEX_URL -> lessNotUnique(ExtensionStoreTable.indexUrl, ExtensionStoreTable.id, cursor, String::toString) + } + + override fun asCursor(type: ExtensionStoreType): Cursor { + val value = + when (this) { + INDEX_URL -> type.indexUrl + NAME -> type.indexUrl + "-" + type.name + } + return Cursor(value) + } + } + + data class ExtensionStoreOrder( + override val by: ExtensionStoreOrderBy, + override val byType: SortOrder? = null, + ) : Order + + data class ExtensionStoreCondition( + val id: Int? = null, + val indexUrl: String? = null, + val name: String? = null, + ) : HasGetOp { + override fun getOp(): Op? { + val opAnd = OpAnd() + opAnd.eq(id, ExtensionStoreTable.id) + opAnd.eq(indexUrl, ExtensionStoreTable.indexUrl) + opAnd.eq(name, ExtensionStoreTable.name) + + return opAnd.op + } + } + + data class ExtensionStoreFilter( + val indexUrl: StringFilter? = null, + val name: StringFilter? = null, + override val and: List? = null, + override val or: List? = null, + override val not: ExtensionStoreFilter? = null, + ) : Filter { + override fun getOpList(): List> = + listOfNotNull( + andFilterWithCompareString(ExtensionStoreTable.indexUrl, indexUrl), + andFilterWithCompareString(ExtensionStoreTable.name, name), + ) + } + + @RequireAuth + fun extensionStores( + condition: ExtensionStoreCondition? = null, + filter: ExtensionStoreFilter? = null, + order: List? = null, + before: Cursor? = null, + after: Cursor? = null, + first: Int? = null, + last: Int? = null, + offset: Int? = null, + ): ExtensionStoreNodeList { + val queryResults = + transaction { + val res = ExtensionStoreTable.selectAll() + + res.applyOps(condition, filter) + + if (order != null || (last != null || before != null)) { + val baseSort = listOf(ExtensionStoreOrder(ExtensionStoreOrderBy.INDEX_URL, SortOrder.ASC)) + val actualSort = (order.orEmpty() + baseSort) + actualSort.forEach { (orderBy, orderByType) -> + val orderByColumn = orderBy.column + val orderType = orderByType.maybeSwap(last ?: before) + + res.orderBy(orderByColumn to orderType) + } + } + + val total = res.count() + val firstResult = res.firstOrNull()?.get(ExtensionStoreTable.indexUrl) + val lastResult = res.lastOrNull()?.get(ExtensionStoreTable.indexUrl) + + res.applyBeforeAfter( + before = before, + after = after, + orderBy = order?.firstOrNull()?.by ?: ExtensionStoreOrderBy.INDEX_URL, + orderByType = order?.firstOrNull()?.byType, + ) + + if (first != null) { + res.limit(first).offset(offset?.toLong() ?: 0) + } else if (last != null) { + res.limit(last) + } + + QueryResults(total, firstResult, lastResult, res.toList()) + } + + val getAsCursor: (ExtensionStoreType) -> Cursor = (order?.firstOrNull()?.by ?: ExtensionStoreOrderBy.INDEX_URL)::asCursor + + val resultsAsType = queryResults.results.map { ExtensionStoreType(it) } + + return ExtensionStoreNodeList( + resultsAsType, + if (resultsAsType.isEmpty()) { + emptyList() + } else { + listOfNotNull( + resultsAsType.firstOrNull()?.let { + ExtensionStoreNodeList.ExtensionStoreEdge( + getAsCursor(it), + it, + ) + }, + resultsAsType.lastOrNull()?.let { + ExtensionStoreNodeList.ExtensionStoreEdge( + getAsCursor(it), + it, + ) + }, + ) + }, + pageInfo = + PageInfo( + hasNextPage = queryResults.lastKey != resultsAsType.lastOrNull()?.indexUrl, + hasPreviousPage = queryResults.firstKey != resultsAsType.firstOrNull()?.indexUrl, + startCursor = resultsAsType.firstOrNull()?.let { getAsCursor(it) }, + endCursor = resultsAsType.lastOrNull()?.let { getAsCursor(it) }, + ), + totalCount = queryResults.total.toInt(), + ) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt index 216ee41b4..928177c5e 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt @@ -13,19 +13,22 @@ import graphql.schema.DataFetchingEnvironment import org.jetbrains.exposed.v1.core.Column import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.greater +import org.jetbrains.exposed.v1.core.greaterEq import org.jetbrains.exposed.v1.core.less import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.graphql.directives.RequireAuth import suwayomi.tachidesk.graphql.queries.filter.BooleanFilter +import suwayomi.tachidesk.graphql.queries.filter.ContentWarningFilter import suwayomi.tachidesk.graphql.queries.filter.Filter import suwayomi.tachidesk.graphql.queries.filter.HasGetOp import suwayomi.tachidesk.graphql.queries.filter.LongFilter import suwayomi.tachidesk.graphql.queries.filter.OpAnd import suwayomi.tachidesk.graphql.queries.filter.StringFilter -import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompare import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareEntity +import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareEnum import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareString import suwayomi.tachidesk.graphql.queries.filter.applyOps import suwayomi.tachidesk.graphql.server.primitives.Cursor @@ -39,6 +42,7 @@ import suwayomi.tachidesk.graphql.server.primitives.lessNotUnique import suwayomi.tachidesk.graphql.server.primitives.maybeSwap import suwayomi.tachidesk.graphql.types.SourceNodeList import suwayomi.tachidesk.graphql.types.SourceType +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.table.SourceTable import java.util.concurrent.CompletableFuture @@ -91,14 +95,23 @@ class SourceQuery { val id: Long? = null, val name: String? = null, val lang: String? = null, + @GraphQLDeprecated("replace with contentWarning == ContentRating.MIXED", ReplaceWith("contentWarning")) val isNsfw: Boolean? = null, + val contentWarning: ContentWarning? = null, ) : HasGetOp { override fun getOp(): Op? { val opAnd = OpAnd() opAnd.eq(id, SourceTable.id) opAnd.eq(name, SourceTable.name) opAnd.eq(lang, SourceTable.lang) - opAnd.eq(isNsfw, SourceTable.isNsfw) + opAnd.andWhere(isNsfw) { + if (it) { + SourceTable.contentWarning greaterEq ContentWarning.MIXED.ordinal + } else { + SourceTable.contentWarning less ContentWarning.MIXED.ordinal + } + } + opAnd.andWhere(contentWarning) { SourceTable.contentWarning eq it.ordinal } return opAnd.op } @@ -108,7 +121,9 @@ class SourceQuery { val id: LongFilter? = null, val name: StringFilter? = null, val lang: StringFilter? = null, + @GraphQLDeprecated("replace with contentWarning", ReplaceWith("contentWarning")) val isNsfw: BooleanFilter? = null, + val contentWarning: ContentWarningFilter? = null, override val and: List? = null, override val or: List? = null, override val not: SourceFilter? = null, @@ -118,7 +133,7 @@ class SourceQuery { andFilterWithCompareEntity(SourceTable.id, id), andFilterWithCompareString(SourceTable.name, name), andFilterWithCompareString(SourceTable.lang, lang), - andFilterWithCompare(SourceTable.isNsfw, isNsfw), + andFilterWithCompareEnum(SourceTable.contentWarning, contentWarning), ) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt index 74998b6c1..98ca11b5d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt @@ -28,6 +28,7 @@ import org.jetbrains.exposed.v1.core.upperCase import org.jetbrains.exposed.v1.core.wrap import org.jetbrains.exposed.v1.jdbc.Query import org.jetbrains.exposed.v1.jdbc.andWhere +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning class ILikeEscapeOp( expr1: Expression<*>, @@ -329,6 +330,24 @@ data class DoubleFilter( ) } +data class ContentWarningFilter( + override val isNull: Boolean? = null, + override val equalTo: ContentWarning? = null, + override val notEqualTo: ContentWarning? = null, + override val notEqualToAll: List? = null, + override val notEqualToAny: List? = null, + override val distinctFrom: ContentWarning? = null, + override val distinctFromAll: List? = null, + override val distinctFromAny: List? = null, + override val notDistinctFrom: ContentWarning? = null, + override val `in`: List? = null, + override val notIn: List? = null, + override val lessThan: ContentWarning? = null, + override val lessThanOrEqualTo: ContentWarning? = null, + override val greaterThan: ContentWarning? = null, + override val greaterThanOrEqualTo: ContentWarning? = null, +) : ComparableScalarFilter + data class StringFilter( override val isNull: Boolean? = null, override val equalTo: String? = null, @@ -618,6 +637,35 @@ fun , S : T?> andFilterWithCompare( return opAnd.op } +@Suppress("UNCHECKED_CAST") +fun > andFilterWithCompareEnum( + column: Column, + filter: ComparableScalarFilter?, +): Op? { + filter ?: return null + val opAnd = OpAnd() + + opAnd.andWhere(filter.lessThan) { column less it.ordinal } + opAnd.andWhere(filter.lessThanOrEqualTo) { column lessEq it.ordinal } + opAnd.andWhere(filter.greaterThan) { column greater it.ordinal } + opAnd.andWhere(filter.greaterThanOrEqualTo) { column greaterEq it.ordinal } + + opAnd.andWhere(filter.isNull) { if (it) column.isNull() else column.isNotNull() } + + opAnd.andWhere(filter.equalTo) { column eq it.ordinal } + opAnd.andNotWhere(filter.notEqualTo, filter.notEqualToAll, filter.notEqualToAny) { column neq it.ordinal } + opAnd.andWhere(filter.distinctFrom, filter.distinctFromAll, filter.distinctFromAny) { DistinctFromOp.distinctFrom(column, it.ordinal) } + opAnd.andWhere(filter.notDistinctFrom) { DistinctFromOp.notDistinctFrom(column, it.ordinal) } + if (!filter.`in`.isNullOrEmpty()) { + opAnd.andWhere(filter.`in`) { column inList it.map { it.ordinal } } + } + if (!filter.notIn.isNullOrEmpty()) { + opAnd.andWhere(filter.notIn) { column notInList it.map { it.ordinal } } + } + + return opAnd.op +} + fun > andFilterWithCompareEntity( column: Column>, filter: ComparableScalarFilter?, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt index 4ff568eea..2f4e28b91 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt @@ -21,6 +21,8 @@ import suwayomi.tachidesk.graphql.dataLoaders.DisplayScoreForTrackSearchDataLoad import suwayomi.tachidesk.graphql.dataLoaders.DownloadedChapterCountForMangaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.ExtensionDataLoader import suwayomi.tachidesk.graphql.dataLoaders.ExtensionForSourceDataLoader +import suwayomi.tachidesk.graphql.dataLoaders.ExtensionStoreDataLoader +import suwayomi.tachidesk.graphql.dataLoaders.ExtensionsForExtensionStore import suwayomi.tachidesk.graphql.dataLoaders.FirstUnreadChapterForMangaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.GlobalMetaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.HasDuplicateChaptersForMangaDataLoader @@ -78,6 +80,8 @@ class TachideskDataLoaderRegistryFactory { SourceMetaDataLoader(), ExtensionDataLoader(), ExtensionForSourceDataLoader(), + ExtensionsForExtensionStore(), + ExtensionStoreDataLoader(), TrackerDataLoader(), TrackerStatusesDataLoader(), TrackerScoresDataLoader(), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt index d6527ae3a..1ee37b97b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt @@ -20,6 +20,7 @@ import suwayomi.tachidesk.graphql.mutations.CategoryMutation import suwayomi.tachidesk.graphql.mutations.ChapterMutation import suwayomi.tachidesk.graphql.mutations.DownloadMutation import suwayomi.tachidesk.graphql.mutations.ExtensionMutation +import suwayomi.tachidesk.graphql.mutations.ExtensionStoreMutation import suwayomi.tachidesk.graphql.mutations.ImageMutation import suwayomi.tachidesk.graphql.mutations.InfoMutation import suwayomi.tachidesk.graphql.mutations.KoreaderSyncMutation @@ -36,6 +37,7 @@ import suwayomi.tachidesk.graphql.queries.CategoryQuery import suwayomi.tachidesk.graphql.queries.ChapterQuery import suwayomi.tachidesk.graphql.queries.DownloadQuery import suwayomi.tachidesk.graphql.queries.ExtensionQuery +import suwayomi.tachidesk.graphql.queries.ExtensionStoreQuery import suwayomi.tachidesk.graphql.queries.InfoQuery import suwayomi.tachidesk.graphql.queries.KoreaderSyncQuery import suwayomi.tachidesk.graphql.queries.MangaQuery @@ -95,6 +97,7 @@ val schema = TopLevelObject(ChapterQuery()), TopLevelObject(DownloadQuery()), TopLevelObject(ExtensionQuery()), + TopLevelObject(ExtensionStoreQuery()), TopLevelObject(InfoQuery()), TopLevelObject(KoreaderSyncQuery()), TopLevelObject(MangaQuery()), @@ -112,6 +115,7 @@ val schema = TopLevelObject(ChapterMutation()), TopLevelObject(DownloadMutation()), TopLevelObject(ExtensionMutation()), + TopLevelObject(ExtensionStoreMutation()), TopLevelObject(ImageMutation()), TopLevelObject(InfoMutation()), TopLevelObject(KoreaderSyncMutation()), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt new file mode 100644 index 000000000..da794a7db --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt @@ -0,0 +1,86 @@ +package suwayomi.tachidesk.graphql.types + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import com.expediagroup.graphql.server.extensions.getValueFromDataLoader +import graphql.schema.DataFetchingEnvironment +import org.jetbrains.exposed.v1.core.ResultRow +import suwayomi.tachidesk.graphql.server.primitives.Cursor +import suwayomi.tachidesk.graphql.server.primitives.Edge +import suwayomi.tachidesk.graphql.server.primitives.Node +import suwayomi.tachidesk.graphql.server.primitives.NodeList +import suwayomi.tachidesk.graphql.server.primitives.PageInfo +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import java.util.concurrent.CompletableFuture + +class ExtensionStoreType( + val name: String, + val badgeLabel: String, + val signingKey: String, + val contactWebsite: String, + val contactDiscord: String?, + val indexUrl: String, + val isLegacy: Boolean, + val extensionListUrl: String?, +) : Node { + constructor(row: ResultRow) : this( + name = row[ExtensionStoreTable.name], + badgeLabel = row[ExtensionStoreTable.badgeLabel], + signingKey = row[ExtensionStoreTable.signingKey], + contactWebsite = row[ExtensionStoreTable.contactWebsite], + contactDiscord = row[ExtensionStoreTable.contactDiscord], + indexUrl = row[ExtensionStoreTable.indexUrl], + isLegacy = row[ExtensionStoreTable.isLegacy], + extensionListUrl = row[ExtensionStoreTable.extensionListUrl], + ) + + fun extensions(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = + dataFetchingEnvironment.getValueFromDataLoader("ExtensionsForExtensionStore", indexUrl) +} + +data class ExtensionStoreNodeList( + override val nodes: List, + override val edges: List, + override val pageInfo: PageInfo, + override val totalCount: Int, +) : NodeList() { + data class ExtensionStoreEdge( + override val cursor: Cursor, + override val node: ExtensionStoreType, + ) : Edge() + + companion object { + fun List.toNodeList(): ExtensionStoreNodeList = + ExtensionStoreNodeList( + nodes = this, + edges = getEdges(), + pageInfo = + PageInfo( + hasNextPage = false, + hasPreviousPage = false, + startCursor = Cursor(0.toString()), + endCursor = Cursor(lastIndex.toString()), + ), + totalCount = size, + ) + + private fun List.getEdges(): List { + if (isEmpty()) return emptyList() + return listOf( + ExtensionStoreEdge( + cursor = Cursor("0"), + node = first(), + ), + ExtensionStoreEdge( + cursor = Cursor(lastIndex.toString()), + node = last(), + ), + ) + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt index 97d6a59ac..cbddc8e59 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt @@ -7,6 +7,8 @@ package suwayomi.tachidesk.graphql.types +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated +import com.expediagroup.graphql.generator.annotations.GraphQLDescription import com.expediagroup.graphql.server.extensions.getValueFromDataLoader import graphql.schema.DataFetchingEnvironment import org.jetbrains.exposed.v1.core.ResultRow @@ -16,33 +18,51 @@ import suwayomi.tachidesk.graphql.server.primitives.Node import suwayomi.tachidesk.graphql.server.primitives.NodeList import suwayomi.tachidesk.graphql.server.primitives.PageInfo import suwayomi.tachidesk.manga.impl.extension.Extension +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.table.ExtensionTable import java.util.concurrent.CompletableFuture class ExtensionType( + val storeIndexUrl: String?, + @GraphQLDeprecated("Removed in extension api v1.6", ReplaceWith("storeIndexUrl")) val repo: String?, - val apkName: String, + @GraphQLDescription("This will be nullable in the future") + val apkName: String?, val iconUrl: String, val name: String, val pkgName: String, + val apkUrl: String?, + val extensionLib: String?, val versionName: String, + @GraphQLDeprecated( + "Type was changed to Long, will be switched back to this variable name in the future.", + ReplaceWith("versionCodeLong"), + ) val versionCode: Int, + val versionCodeLong: Long, val lang: String, + @GraphQLDeprecated("Removed in extension api v1.6", ReplaceWith("contentWarning")) val isNsfw: Boolean, + val contentWarning: ContentWarning, val isInstalled: Boolean, val hasUpdate: Boolean, val isObsolete: Boolean, ) : Node { constructor(row: ResultRow) : this( - repo = row[ExtensionTable.repo], - apkName = row[ExtensionTable.apkName], - iconUrl = Extension.getExtensionIconUrl(row[ExtensionTable.apkName]), + storeIndexUrl = row[ExtensionTable.storeIndexUrl], + repo = row[ExtensionTable.storeIndexUrl], + apkName = row[ExtensionTable.apkName].orEmpty(), + iconUrl = Extension.proxyExtensionIconUrl(row[ExtensionTable.pkgName]), name = row[ExtensionTable.name], pkgName = row[ExtensionTable.pkgName], + apkUrl = row[ExtensionTable.apkUrl], + extensionLib = row[ExtensionTable.extensionLib], versionName = row[ExtensionTable.versionName], - versionCode = row[ExtensionTable.versionCode], + versionCode = row[ExtensionTable.versionCode].toInt(), + versionCodeLong = row[ExtensionTable.versionCode], lang = row[ExtensionTable.lang], - isNsfw = row[ExtensionTable.isNsfw], + isNsfw = row[ExtensionTable.contentWarning] >= ContentWarning.MIXED.ordinal, + contentWarning = ContentWarning.valueOf(row[ExtensionTable.contentWarning]), isInstalled = row[ExtensionTable.isInstalled], hasUpdate = row[ExtensionTable.hasUpdate], isObsolete = row[ExtensionTable.isObsolete], @@ -50,6 +70,9 @@ class ExtensionType( fun source(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = dataFetchingEnvironment.getValueFromDataLoader("SourcesForExtensionDataLoader", pkgName) + + fun extensionStore(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = + dataFetchingEnvironment.getValueFromDataLoader("ExtensionStoreDataLoader", storeIndexUrl.orEmpty()) } data class ExtensionNodeList( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt index f6c4744a2..94d41b58e 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt @@ -7,6 +7,7 @@ package suwayomi.tachidesk.graphql.types +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated import com.expediagroup.graphql.server.extensions.getValueFromDataLoader import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.ConfigurableSource @@ -25,7 +26,7 @@ import suwayomi.tachidesk.manga.impl.Source.getSourcePreferencesRaw import suwayomi.tachidesk.manga.impl.extension.Extension import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub -import suwayomi.tachidesk.manga.model.dataclass.SourceDataClass +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.table.ExtensionTable import suwayomi.tachidesk.manga.model.table.SourceTable import java.util.concurrent.CompletableFuture @@ -41,35 +42,29 @@ class SourceType( val id: Long, val name: String, val lang: String, + val contentWarning: ContentWarning, val iconUrl: String, val supportsLatest: Boolean, val isConfigurable: Boolean, + @GraphQLDeprecated("", ReplaceWith("contentWarning")) val isNsfw: Boolean, val displayName: String, + val homeUrl: String?, + @GraphQLDeprecated("", ReplaceWith("homeUrl")) val baseUrl: String?, ) : Node { - constructor(source: SourceDataClass) : this( - id = source.id.toLong(), - name = source.name, - lang = source.lang, - iconUrl = source.iconUrl, - supportsLatest = source.supportsLatest, - isConfigurable = source.isConfigurable, - isNsfw = source.isNsfw, - displayName = source.displayName, - baseUrl = source.baseUrl, - ) - constructor(row: ResultRow, sourceExtension: ResultRow, catalogueSource: CatalogueSource) : this( id = row[SourceTable.id].value, name = row[SourceTable.name], lang = row[SourceTable.lang], - iconUrl = Extension.getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]), + contentWarning = ContentWarning.valueOf(row[SourceTable.contentWarning]), + iconUrl = Extension.proxyExtensionIconUrl(sourceExtension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = row[SourceTable.isNsfw], + isNsfw = row[SourceTable.contentWarning] >= ContentWarning.MIXED.ordinal, displayName = catalogueSource.toString(), - baseUrl = catalogueSource.runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), + homeUrl = runCatching { (catalogueSource as? HttpSource)?.getHomeUrl() }.getOrNull(), + baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) fun manga(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt index 89b8f3dfc..2da3c3740 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt @@ -34,7 +34,7 @@ object MangaAPI { get("update/{pkgName}", ExtensionController.update) get("uninstall/{pkgName}", ExtensionController.uninstall) - get("icon/{apkName}", ExtensionController.icon) + get("icon/{pkgName}", ExtensionController.icon) } path("source") { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt index 006b27adb..220864dab 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt @@ -165,17 +165,17 @@ object ExtensionController { /** icon for extension named `apkName` */ val icon = handler( - pathParam("apkName"), + pathParam("pkgName"), documentWith = { withOperation { summary("Extension icon") description("Icon for extension named `apkName`") } }, - behaviorOf = { ctx, apkName -> + behaviorOf = { ctx, pkgName -> ctx.getAttribute(Attribute.TachideskUser).requireUser() ctx.future { - future { Extension.getExtensionIcon(apkName) } + future { Extension.getExtensionIcon(pkgName) } .thenApply { ctx.header("content-type", it.second) val httpCacheSeconds = 365.days.inWholeSeconds diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt index 1fe1559ae..0b5871843 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt @@ -55,7 +55,7 @@ object UpdateController { ) /** - * Class made for handling return type in the documentation for [recentChapters], + * Class made for handling return type in the documentation for [UpdateController.recentChapters], * since OpenApi cannot handle runtime generics. */ private class PagedMangaChapterListDataClass : PaginatedList(emptyList(), false) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index 763a39989..d2758c324 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -7,16 +7,19 @@ package suwayomi.tachidesk.manga.impl * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.UpdateStrategy import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.util.chapter.ChapterRecognition import eu.kanade.tachiyomi.util.chapter.ChapterSanitizer.sanitize import io.github.oshai.kotlinlogging.KotlinLogging -import io.github.reactivecircus.cache4k.Cache import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.dao.id.EntityID @@ -32,7 +35,6 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.Manga.getManga import suwayomi.tachidesk.manga.impl.download.DownloadManager import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput import suwayomi.tachidesk.manga.impl.track.Track @@ -50,7 +52,6 @@ import suwayomi.tachidesk.server.serverConfig import java.time.Instant import java.util.TreeSet import kotlin.math.max -import kotlin.time.Duration.Companion.minutes private fun List.removeDuplicates(currentChapter: ChapterDataClass): List = groupBy { it.chapterNumber } @@ -104,267 +105,277 @@ object Chapter { .associateBy({ it[ChapterTable.url] }, { it }) } - return chapterList.mapIndexed { index, it -> - + return chapterList.map { val dbChapter = dbChapterMap.getValue(it.url) - - ChapterDataClass( - id = dbChapter[ChapterTable.id].value, - url = it.url, - name = it.name, - uploadDate = it.date_upload, - chapterNumber = it.chapter_number, - scanlator = it.scanlator, - mangaId = mangaId, - read = dbChapter[ChapterTable.isRead], - bookmarked = dbChapter[ChapterTable.isBookmarked], - lastPageRead = dbChapter[ChapterTable.lastPageRead], - lastReadAt = dbChapter[ChapterTable.lastReadAt], - index = chapterList.size - index, - fetchedAt = dbChapter[ChapterTable.fetchedAt], - realUrl = dbChapter[ChapterTable.realUrl], - downloaded = dbChapter[ChapterTable.isDownloaded], - pageCount = dbChapter[ChapterTable.pageCount], - lastModifiedAt = dbChapter[ChapterTable.lastModifiedAt], - version = dbChapter[ChapterTable.version], - ) + ChapterTable.toDataClass(dbChapter) } } - val map: Cache = - Cache - .Builder() - .expireAfterAccess(10.minutes) - .build() - suspend fun fetchChapterList(mangaId: Int): List { - val mutex = map.get(mangaId) { Mutex() } + val mutex = Manga.mangaInfoMutex.get(mangaId) { Mutex() } val chapterList = mutex.withLock { - val manga = getManga(mangaId) - val source = getCatalogueSourceOrStub(manga.sourceId.toLong()) - - val sManga = - SManga.create().apply { - title = manga.title - url = manga.url - description = manga.description - } - - val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f - val numberOfCurrentChapters = getCountOfMangaChapters(mangaId) - - val chapters = source.getChapterList(sManga) - // it's possible that the source returns a list containing chapters with the same url - // once such duplicated chapters have been added, they aren't being removed anymore as long as there is - // a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself - // does not exist anymore on the source - val uniqueChapters = chapters.distinctBy { it.url } - - if (uniqueChapters.isEmpty()) { - throw Exception("No chapters found") - } - - // Recognize number for new chapters. - uniqueChapters.forEach { chapter -> - (source as? HttpSource)?.prepareNewChapter(chapter, sManga) - val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble()) - chapter.chapter_number = chapterNumber.toFloat() - chapter.name = chapter.name.sanitize(manga.title) - chapter.scanlator = chapter.scanlator?.ifBlank { null }?.trim() - } - - val now = Instant.now().epochSecond - // Used to not set upload date of older chapters - // to a higher value than newer chapters - var maxSeenUploadDate = 0L - - val chaptersInDb = + val mangaEntry = transaction { - ChapterTable - .selectAll() - .where { ChapterTable.manga eq mangaId } - .map { ChapterTable.toDataClass(it) } - .toList() + MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } + val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) - // new chapters after they have been added to the database for auto downloads - val insertedChapterIds = mutableListOf() + val chapters = + Manga + .fetchMangaAndChapters( + mangaEntry = mangaEntry, + source = source, + fetchDetails = false, + fetchChapters = true, + ).chapters - val chaptersToInsert = mutableListOf() // do not yet have an ID from the database - val chaptersToUpdate = mutableListOf() - - uniqueChapters.reversed().forEachIndexed { index, fetchedChapter -> - val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url } - - val chapterData = - ChapterDataClass.fromSChapter( - fetchedChapter, - chapterEntry?.id ?: 0, - index + 1, - now, - mangaId, - runCatching { - (source as? HttpSource)?.getChapterUrl(fetchedChapter) - }.getOrNull(), - ) - - if (chapterEntry == null) { - val newChapterData = - if (chapterData.uploadDate == 0L) { - val altDateUpload = if (maxSeenUploadDate == 0L) now else maxSeenUploadDate - chapterData.copy(uploadDate = altDateUpload) - } else { - maxSeenUploadDate = max(maxSeenUploadDate, chapterData.uploadDate) - chapterData - } - chaptersToInsert.add(newChapterData) - } else { - val newChapterData = - if (chapterData.uploadDate == 0L) { - chapterData.copy(uploadDate = chapterEntry.uploadDate) - } else { - chapterData - } - chaptersToUpdate.add(newChapterData) - } - } - - val deletedChapterNumbers = TreeSet() - val deletedReadChapterNumbers = TreeSet() - val deletedBookmarkedChapterNumbers = TreeSet() - val deletedDownloadedChapterNumberToChapter = mutableMapOf() - val deletedChapterNumberDateFetchMap = mutableMapOf() - - // clear any orphaned/duplicate chapters that are in the db but not in `chapterList` - val chapterUrls = uniqueChapters.map { it.url }.toSet() - - val chaptersIdsToDelete = - chaptersInDb.mapNotNull { dbChapter -> - if (!chapterUrls.contains(dbChapter.url)) { - if (dbChapter.read) deletedReadChapterNumbers.add(dbChapter.chapterNumber) - if (dbChapter.bookmarked) deletedBookmarkedChapterNumbers.add(dbChapter.chapterNumber) - if (dbChapter.downloaded) deletedDownloadedChapterNumberToChapter[dbChapter.chapterNumber] = dbChapter - deletedChapterNumbers.add(dbChapter.chapterNumber) - deletedChapterNumberDateFetchMap[dbChapter.chapterNumber] = dbChapter.fetchedAt - dbChapter.id - } else { - null - } - } - - transaction { - // we got some clean up due - if (chaptersIdsToDelete.isNotEmpty()) { - DownloadManager.dequeue(chaptersIdsToDelete) - PageTable.deleteWhere { chapter inList chaptersIdsToDelete } - ChapterTable.deleteWhere { id inList chaptersIdsToDelete } - } - - if (chaptersToInsert.isNotEmpty()) { - ChapterTable - .batchInsert(chaptersToInsert) { chapter -> - this[ChapterTable.url] = chapter.url - this[ChapterTable.name] = chapter.name - this[ChapterTable.date_upload] = chapter.uploadDate - this[ChapterTable.chapter_number] = chapter.chapterNumber - this[ChapterTable.scanlator] = chapter.scanlator - this[ChapterTable.sourceOrder] = chapter.index - this[ChapterTable.fetchedAt] = chapter.fetchedAt - this[ChapterTable.manga] = chapter.mangaId - this[ChapterTable.realUrl] = chapter.realUrl - this[ChapterTable.isRead] = false - this[ChapterTable.isBookmarked] = false - this[ChapterTable.isDownloaded] = false - this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt - this[ChapterTable.version] = chapter.version - this[ChapterTable.pageCount] = -1 - - // is recognized chapter number - if (chapter.chapterNumber >= 0f && chapter.chapterNumber in deletedChapterNumbers) { - this[ChapterTable.isRead] = chapter.chapterNumber in deletedReadChapterNumbers - this[ChapterTable.isBookmarked] = chapter.chapterNumber in deletedBookmarkedChapterNumbers - - // Try to use the fetch date of the original entry to not pollute 'Updates' tab - deletedChapterNumberDateFetchMap[chapter.chapterNumber]?.let { - this[ChapterTable.fetchedAt] = it - } - - deletedDownloadedChapterNumberToChapter[chapter.chapterNumber]?.let { - val hasDownloadedPages = it.pageCount > 0 - val isSameName = it.name == chapter.name - val isSameScanlator = it.scanlator == chapter.scanlator - - // Only preserve download status for chapters with the same name and of the same scanlator; otherwise, - // the downloaded files won't be found anyway - val isDownloadPreservable = hasDownloadedPages && isSameName && isSameScanlator - if (isDownloadPreservable) { - this[ChapterTable.isDownloaded] = true - this[ChapterTable.pageCount] = it.pageCount - } - } - } - }.forEach { insertedChapterIds.add(it[ChapterTable.id].value) } - } - - if (chaptersToUpdate.isNotEmpty()) { - BatchUpdateStatement(ChapterTable) - .apply { - chaptersToUpdate.forEach { - addBatch(EntityID(it.id, ChapterTable)) - - val currentChapter = chaptersInDb.find { dbChapter -> dbChapter.id == it.id }!! - - this[ChapterTable.name] = it.name - this[ChapterTable.date_upload] = it.uploadDate - this[ChapterTable.chapter_number] = it.chapterNumber - this[ChapterTable.scanlator] = it.scanlator - this[ChapterTable.sourceOrder] = it.index - this[ChapterTable.realUrl] = it.realUrl - this[ChapterTable.lastModifiedAt] = it.lastModifiedAt - this[ChapterTable.version] = it.version - this[ChapterTable.isDownloaded] = currentChapter.downloaded - this[ChapterTable.pageCount] = currentChapter.pageCount - - if (!currentChapter.downloaded) { - return@forEach - } - - val isSameScanlator = currentChapter.scanlator == it.scanlator - val isSameName = currentChapter.name == it.name - - val isDownloadPreservable = isSameName && isSameScanlator - if (!isDownloadPreservable) { - this[ChapterTable.isDownloaded] = false - this[ChapterTable.pageCount] = -1 - } - } - }.toExecutable() - .execute(this@transaction) - } - - MangaTable.update({ MangaTable.id eq mangaId }) { - it[chaptersLastFetchedAt] = Instant.now().epochSecond - } - } - - if (manga.inLibrary) { - // We have to query the inserted chapters to get the up-to-date data. I.e. "last_modified_at" is not returned by the insert statement, due to being set by a DB trigger - val insertedChapters = - transaction { - ChapterTable.selectAll().where { ChapterTable.id inList insertedChapterIds }.map( - ChapterTable::toDataClass, - ) - } - downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters) - } - - uniqueChapters + updateChapterListDatabase(mangaEntry, chapters, source) } return chapterList } + fun updateChapterListDatabase( + mangaEntry: ResultRow, + chapters: List, + source: CatalogueSource, + ): List { + val currentLatestChapterNumber = Manga.getLatestChapter(mangaEntry[MangaTable.id].value)?.chapterNumber ?: 0f + val numberOfCurrentChapters = getCountOfMangaChapters(mangaEntry[MangaTable.id].value) + // it's possible that the source returns a list containing chapters with the same url + // once such duplicated chapters have been added, they aren't being removed anymore as long as there is + // a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself + // does not exist anymore on the source + val uniqueChapters = chapters.distinctBy { it.url } + + if (uniqueChapters.isEmpty()) { + throw Exception("No chapters found") + } + + // Recognize number for new chapters. + val sManga = + SManga.create().apply { + url = mangaEntry[MangaTable.url] + title = mangaEntry[MangaTable.title] + thumbnail_url = mangaEntry[MangaTable.thumbnail_url] + artist = mangaEntry[MangaTable.artist] + author = mangaEntry[MangaTable.author] + description = mangaEntry[MangaTable.description] + genre = mangaEntry[MangaTable.genre] + status = mangaEntry[MangaTable.status] + update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) + memo = Json.decodeFromString(mangaEntry[MangaTable.memo]) + initialized = mangaEntry[MangaTable.initialized] + } + uniqueChapters.forEach { chapter -> + (source as? HttpSource)?.prepareNewChapter(chapter, sManga) + val chapterNumber = + ChapterRecognition.parseChapterNumber( + mangaEntry[MangaTable.title], + chapter.name, + chapter.chapter_number.toDouble(), + ) + chapter.chapter_number = chapterNumber.toFloat() + chapter.name = chapter.name.sanitize(mangaEntry[MangaTable.title]) + chapter.scanlator = chapter.scanlator?.ifBlank { null }?.trim() + } + + val now = Instant.now().epochSecond + // Used to not set upload date of older chapters + // to a higher value than newer chapters + var maxSeenUploadDate = 0L + + val chaptersInDb = + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq mangaEntry[MangaTable.id].value } + .map { ChapterTable.toDataClass(it) } + .toList() + } + + // new chapters after they have been added to the database for auto downloads + val insertedChapterIds = mutableListOf() + + val chaptersToInsert = mutableListOf() // do not yet have an ID from the database + val chaptersToUpdate = mutableListOf() + + uniqueChapters.reversed().forEachIndexed { index, fetchedChapter -> + val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url } + + val chapterData = + ChapterDataClass.fromSChapter( + fetchedChapter, + chapterEntry?.id ?: 0, + index + 1, + now, + mangaEntry[MangaTable.id].value, + runCatching { + (source as? HttpSource)?.getChapterUrl(fetchedChapter) + }.getOrNull(), + ) + + if (chapterEntry == null) { + val newChapterData = + if (chapterData.uploadDate == 0L) { + val altDateUpload = if (maxSeenUploadDate == 0L) now else maxSeenUploadDate + chapterData.copy(uploadDate = altDateUpload) + } else { + maxSeenUploadDate = max(maxSeenUploadDate, chapterData.uploadDate) + chapterData + } + chaptersToInsert.add(newChapterData) + } else { + val newChapterData = + if (chapterData.uploadDate == 0L) { + chapterData.copy(uploadDate = chapterEntry.uploadDate) + } else { + chapterData + } + chaptersToUpdate.add(newChapterData) + } + } + + val deletedChapterNumbers = TreeSet() + val deletedReadChapterNumbers = TreeSet() + val deletedBookmarkedChapterNumbers = TreeSet() + val deletedDownloadedChapterNumberToChapter = mutableMapOf() + val deletedChapterNumberDateFetchMap = mutableMapOf() + + // clear any orphaned/duplicate chapters that are in the db but not in `chapterList` + val chapterUrls = uniqueChapters.map { it.url }.toSet() + + val chaptersIdsToDelete = + chaptersInDb.mapNotNull { dbChapter -> + if (!chapterUrls.contains(dbChapter.url)) { + if (dbChapter.read) deletedReadChapterNumbers.add(dbChapter.chapterNumber) + if (dbChapter.bookmarked) deletedBookmarkedChapterNumbers.add(dbChapter.chapterNumber) + if (dbChapter.downloaded) deletedDownloadedChapterNumberToChapter[dbChapter.chapterNumber] = dbChapter + deletedChapterNumbers.add(dbChapter.chapterNumber) + deletedChapterNumberDateFetchMap[dbChapter.chapterNumber] = dbChapter.fetchedAt + dbChapter.id + } else { + null + } + } + + transaction { + // we got some clean up due + if (chaptersIdsToDelete.isNotEmpty()) { + DownloadManager.dequeue(chaptersIdsToDelete) + PageTable.deleteWhere { chapter inList chaptersIdsToDelete } + ChapterTable.deleteWhere { id inList chaptersIdsToDelete } + } + + if (chaptersToInsert.isNotEmpty()) { + ChapterTable + .batchInsert(chaptersToInsert) { chapter -> + this[ChapterTable.url] = chapter.url + this[ChapterTable.name] = chapter.name + this[ChapterTable.date_upload] = chapter.uploadDate + this[ChapterTable.chapter_number] = chapter.chapterNumber + this[ChapterTable.scanlator] = chapter.scanlator + this[ChapterTable.sourceOrder] = chapter.index + this[ChapterTable.fetchedAt] = chapter.fetchedAt + this[ChapterTable.manga] = chapter.mangaId + this[ChapterTable.realUrl] = chapter.realUrl + this[ChapterTable.memo] = Json.encodeToString(chapter.memo) + this[ChapterTable.isRead] = false + this[ChapterTable.isBookmarked] = false + this[ChapterTable.isDownloaded] = false + this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt + this[ChapterTable.version] = chapter.version + this[ChapterTable.pageCount] = -1 + + // is recognized chapter number + if (chapter.chapterNumber >= 0f && chapter.chapterNumber in deletedChapterNumbers) { + this[ChapterTable.isRead] = chapter.chapterNumber in deletedReadChapterNumbers + this[ChapterTable.isBookmarked] = chapter.chapterNumber in deletedBookmarkedChapterNumbers + + // Try to use the fetch date of the original entry to not pollute 'Updates' tab + deletedChapterNumberDateFetchMap[chapter.chapterNumber]?.let { + this[ChapterTable.fetchedAt] = it + } + + deletedDownloadedChapterNumberToChapter[chapter.chapterNumber]?.let { + val hasDownloadedPages = it.pageCount > 0 + val isSameName = it.name == chapter.name + val isSameScanlator = it.scanlator == chapter.scanlator + + // Only preserve download status for chapters with the same name and of the same scanlator; otherwise, + // the downloaded files won't be found anyway + val isDownloadPreservable = hasDownloadedPages && isSameName && isSameScanlator + if (isDownloadPreservable) { + this[ChapterTable.isDownloaded] = true + this[ChapterTable.pageCount] = it.pageCount + } + } + } + }.forEach { insertedChapterIds.add(it[ChapterTable.id].value) } + } + + if (chaptersToUpdate.isNotEmpty()) { + BatchUpdateStatement(ChapterTable) + .apply { + chaptersToUpdate.forEach { + addBatch(EntityID(it.id, ChapterTable)) + + val currentChapter = chaptersInDb.find { dbChapter -> dbChapter.id == it.id }!! + + this[ChapterTable.name] = it.name + this[ChapterTable.date_upload] = it.uploadDate + this[ChapterTable.chapter_number] = it.chapterNumber + this[ChapterTable.scanlator] = it.scanlator + this[ChapterTable.sourceOrder] = it.index + this[ChapterTable.realUrl] = it.realUrl + this[ChapterTable.lastModifiedAt] = it.lastModifiedAt + this[ChapterTable.version] = it.version + this[ChapterTable.memo] = Json.encodeToString(it.memo) + this[ChapterTable.isDownloaded] = currentChapter.downloaded + this[ChapterTable.pageCount] = currentChapter.pageCount + + if (!currentChapter.downloaded) { + return@forEach + } + + val isSameScanlator = currentChapter.scanlator == it.scanlator + val isSameName = currentChapter.name == it.name + + val isDownloadPreservable = isSameName && isSameScanlator + if (!isDownloadPreservable) { + this[ChapterTable.isDownloaded] = false + this[ChapterTable.pageCount] = -1 + } + } + }.toExecutable() + .execute(this@transaction) + } + + MangaTable.update({ MangaTable.id eq mangaEntry[MangaTable.id].value }) { + it[chaptersLastFetchedAt] = Instant.now().epochSecond + } + } + + if (mangaEntry[MangaTable.inLibrary]) { + // We have to query the inserted chapters to get the up-to-date data. I.e. "last_modified_at" is not returned by the insert statement, due to being set by a DB trigger + val insertedChapters = + transaction { + ChapterTable.selectAll().where { ChapterTable.id inList insertedChapterIds }.map( + ChapterTable::toDataClass, + ) + } + downloadNewChapters( + mangaEntry[MangaTable.id].value, + currentLatestChapterNumber, + numberOfCurrentChapters, + insertedChapters, + ) + } + + return uniqueChapters + } + private fun downloadNewChapters( mangaId: Int, prevLatestChapterNumber: Float, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index 90b14996e..9d31f7fa2 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -11,13 +11,20 @@ import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.HttpException import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.network.awaitSuccess +import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.local.LocalSource +import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.SMangaUpdate import eu.kanade.tachiyomi.source.model.UpdateStrategy import eu.kanade.tachiyomi.source.online.HttpSource import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging +import io.github.reactivecircus.cache4k.Cache import io.javalin.http.HttpStatus +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.json.Json import okhttp3.CacheControl import okhttp3.Response import org.jetbrains.exposed.v1.core.ResultRow @@ -32,10 +39,7 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl -import suwayomi.tachidesk.manga.impl.Source.getSource import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.MissingThumbnailException -import suwayomi.tachidesk.manga.impl.track.Track import suwayomi.tachidesk.manga.impl.util.network.await import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub @@ -47,10 +51,8 @@ import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass -import suwayomi.tachidesk.manga.model.dataclass.toGenreList import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.manga.model.table.MangaMetaTable -import suwayomi.tachidesk.manga.model.table.MangaStatus import suwayomi.tachidesk.manga.model.table.MangaTable import suwayomi.tachidesk.manga.model.table.toDataClass import suwayomi.tachidesk.server.ApplicationDirs @@ -59,10 +61,17 @@ import java.io.File import java.io.IOException import java.io.InputStream import java.time.Instant +import kotlin.time.Duration.Companion.minutes private val logger = KotlinLogging.logger { } object Manga { + val mangaInfoMutex: Cache = + Cache + .Builder() + .expireAfterAccess(10.minutes) + .build() + suspend fun getManga( mangaId: Int, onlineFetch: Boolean = false, @@ -70,63 +79,118 @@ object Manga { var mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } return if (!onlineFetch && mangaEntry[MangaTable.initialized]) { - getMangaDataClass(mangaId, mangaEntry) + MangaTable.toDataClass(mangaEntry) } else { // initialize manga - val sManga = fetchManga(mangaId) ?: return getMangaDataClass(mangaId, mangaEntry) + updateMangaAndChapters(mangaId, updateChapters = false) mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } - MangaDataClass( - id = mangaId, - sourceId = mangaEntry[MangaTable.sourceReference].toString(), - url = mangaEntry[MangaTable.url], - title = mangaEntry[MangaTable.title], - thumbnailUrl = proxyThumbnailUrl(mangaId), - thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched], - initialized = true, - artist = sManga.artist, - author = sManga.author, - description = sManga.description, - genre = sManga.genre.toGenreList(), - status = MangaStatus.valueOf(sManga.status).name, - inLibrary = mangaEntry[MangaTable.inLibrary], - inLibraryAt = mangaEntry[MangaTable.inLibraryAt], - source = getSource(mangaEntry[MangaTable.sourceReference]), - realUrl = mangaEntry[MangaTable.realUrl], - lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt], - chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt], - updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), - freshData = true, - trackers = Track.getTrackRecordsByMangaId(mangaId), - lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt], - version = mangaEntry[MangaTable.version], - ) + MangaTable.toDataClass(mangaEntry).copy(freshData = true) } } - suspend fun fetchManga(mangaId: Int): SManga? { - val mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } - - val source = - getCatalogueSourceOrNull(mangaEntry[MangaTable.sourceReference]) - ?: return null + suspend fun fetchMangaAndChapters( + mangaEntry: ResultRow, + source: CatalogueSource, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate { val sManga = - source.getMangaDetails( - SManga.create().apply { - url = mangaEntry[MangaTable.url] - title = mangaEntry[MangaTable.title] - thumbnail_url = mangaEntry[MangaTable.thumbnail_url] - artist = mangaEntry[MangaTable.artist] - author = mangaEntry[MangaTable.author] - description = mangaEntry[MangaTable.description] - genre = mangaEntry[MangaTable.genre] - status = mangaEntry[MangaTable.status] - update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) - }, - ) + SManga.create().apply { + url = mangaEntry[MangaTable.url] + title = mangaEntry[MangaTable.title] + thumbnail_url = mangaEntry[MangaTable.thumbnail_url] + artist = mangaEntry[MangaTable.artist] + author = mangaEntry[MangaTable.author] + description = mangaEntry[MangaTable.description] + genre = mangaEntry[MangaTable.genre] + status = mangaEntry[MangaTable.status] + update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) + memo = Json.decodeFromString(mangaEntry[MangaTable.memo]) + initialized = mangaEntry[MangaTable.initialized] + } + val sChapters = + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq mangaEntry[MangaTable.id] } + .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) + .map { + SChapter.create().apply { + url = it[ChapterTable.url] + name = it[ChapterTable.name] + chapter_number = it[ChapterTable.chapter_number] + scanlator = it[ChapterTable.scanlator] + date_upload = it[ChapterTable.date_upload] + memo = Json.decodeFromString(it[ChapterTable.memo]) + } + } + } + return source.getMangaUpdate( + sManga, + sChapters, + fetchDetails = fetchDetails, + fetchChapters = fetchChapters, + ) + } + + suspend fun fetchManga(mangaId: Int): SManga? { + return mangaInfoMutex.get(mangaId) { Mutex() }.withLock { + val mangaEntry = + transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } + val source = getCatalogueSourceOrNull(mangaEntry[MangaTable.sourceReference]) ?: return null + val sManga = + fetchMangaAndChapters( + mangaEntry, + source, + fetchDetails = true, + fetchChapters = false, + ).manga + + updateMangaDatabase(mangaEntry, source, sManga) + } + } + + suspend fun updateMangaAndChapters( + mangaId: Int, + updateManga: Boolean = true, + updateChapters: Boolean = true, + ) { + mangaInfoMutex.get(mangaId) { Mutex() }.withLock { + var mangaEntry = + transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } + val source = + getCatalogueSourceOrNull(mangaEntry[MangaTable.sourceReference]) + ?: throw NullPointerException("Missing source ${mangaEntry[MangaTable.sourceReference]}") + val mangaUpdate = + fetchMangaAndChapters( + mangaEntry, + source, + fetchDetails = updateManga, + fetchChapters = updateChapters, + ) + + if (updateManga) { + updateMangaDatabase(mangaEntry, source, mangaUpdate.manga) + mangaEntry = + transaction { + MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() + } + } + if (updateChapters) { + Chapter.updateChapterListDatabase(mangaEntry, mangaUpdate.chapters, source) + } + } + } + + fun updateMangaDatabase( + mangaEntry: ResultRow, + source: CatalogueSource, + sManga: SManga, + ): SManga { transaction { - MangaTable.update({ MangaTable.id eq mangaId }) { + MangaTable.update({ MangaTable.id eq mangaEntry[MangaTable.id] }) { val remoteTitle = try { sManga.title @@ -151,7 +215,7 @@ object Manga { if (!sManga.thumbnail_url.isNullOrEmpty()) { it[MangaTable.thumbnail_url] = sManga.thumbnail_url it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond - clearThumbnail(mangaId) + clearThumbnail(mangaEntry[MangaTable.id].value) } it[MangaTable.realUrl] = @@ -221,35 +285,6 @@ object Manga { } } - private fun getMangaDataClass( - mangaId: Int, - mangaEntry: ResultRow, - ) = MangaDataClass( - id = mangaId, - sourceId = mangaEntry[MangaTable.sourceReference].toString(), - url = mangaEntry[MangaTable.url], - title = mangaEntry[MangaTable.title], - thumbnailUrl = proxyThumbnailUrl(mangaId), - thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched], - initialized = true, - artist = mangaEntry[MangaTable.artist], - author = mangaEntry[MangaTable.author], - description = mangaEntry[MangaTable.description], - genre = mangaEntry[MangaTable.genre].toGenreList(), - status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, - inLibrary = mangaEntry[MangaTable.inLibrary], - inLibraryAt = mangaEntry[MangaTable.inLibraryAt], - source = getSource(mangaEntry[MangaTable.sourceReference]), - realUrl = mangaEntry[MangaTable.realUrl], - lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt], - chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt], - updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), - freshData = false, - trackers = Track.getTrackRecordsByMangaId(mangaId), - lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt], - version = mangaEntry[MangaTable.version], - ) - fun getMangaMetaMap(mangaId: Int): Map = transaction { MangaMetaTable diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt index 9d405c506..b7d888ec3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt @@ -9,6 +9,7 @@ package suwayomi.tachidesk.manga.impl import eu.kanade.tachiyomi.source.local.LocalSource import eu.kanade.tachiyomi.source.model.MangasPage +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.dao.id.EntityID import org.jetbrains.exposed.v1.core.eq @@ -75,6 +76,7 @@ object MangaList { this[MangaTable.status] = it.status this[MangaTable.thumbnail_url] = it.thumbnail_url this[MangaTable.updateStrategy] = it.update_strategy.name + this[MangaTable.memo] = Json.encodeToString(it.memo) this[MangaTable.sourceReference] = sourceId }.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) } @@ -103,6 +105,7 @@ object MangaList { this[MangaTable.status] = sManga.status this[MangaTable.thumbnail_url] = sManga.thumbnail_url ?: manga[MangaTable.thumbnail_url] this[MangaTable.updateStrategy] = sManga.update_strategy.name + this[MangaTable.memo] = Json.encodeToString(sManga.memo) if (!sManga.thumbnail_url.isNullOrEmpty() && manga[MangaTable.thumbnail_url] != sManga.thumbnail_url) { this[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond Manga.clearThumbnail(manga[MangaTable.id].value) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt index fbf5e6d12..87db6f6d2 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt @@ -25,10 +25,11 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.manga.impl.Source.preferenceScreenMap -import suwayomi.tachidesk.manga.impl.extension.Extension.getExtensionIconUrl +import suwayomi.tachidesk.manga.impl.extension.Extension.proxyExtensionIconUrl import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.unregisterCatalogueSource +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.dataclass.SourceDataClass import suwayomi.tachidesk.manga.model.table.ExtensionTable import suwayomi.tachidesk.manga.model.table.SourceMetaTable @@ -49,10 +50,10 @@ object Source { id = it[SourceTable.id].value.toString(), name = it[SourceTable.name], lang = it[SourceTable.lang], - iconUrl = getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]), + iconUrl = proxyExtensionIconUrl(sourceExtension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = it[SourceTable.isNsfw], + isNsfw = it[SourceTable.contentWarning] >= ContentWarning.MIXED.ordinal, displayName = catalogueSource.toString(), baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) @@ -70,13 +71,10 @@ object Source { id = sourceId.toString(), name = source[SourceTable.name], lang = source[SourceTable.lang], - iconUrl = - getExtensionIconUrl( - extension[ExtensionTable.apkName], - ), + iconUrl = proxyExtensionIconUrl(extension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = source[SourceTable.isNsfw], + isNsfw = source[SourceTable.contentWarning] >= ContentWarning.MIXED.ordinal, displayName = catalogueSource.toString(), baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt index 6e4c26587..02343faba 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt @@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl.backup.proto.handlers * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import eu.kanade.tachiyomi.source.model.UpdateStrategy +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and @@ -76,6 +77,8 @@ object BackupMangaHandler { updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]), lastModifiedAt = mangaRow[MangaTable.lastModifiedAt], version = mangaRow[MangaTable.version], + initialized = mangaRow[MangaTable.initialized], + memo = Json.encodeToString(mangaRow[MangaTable.memo]).encodeToByteArray(), ) val mangaId = mangaRow[MangaTable.id].value @@ -238,6 +241,7 @@ object BackupMangaHandler { it[lastModifiedAt] = manga.lastModifiedAt it[version] = manga.version + it[memo] = manga.memo.decodeToString() }.value } else { val dbMangaId = dbManga[MangaTable.id].value @@ -260,6 +264,7 @@ object BackupMangaHandler { it[lastModifiedAt] = manga.lastModifiedAt it[version] = manga.version + it[memo] = manga.memo.decodeToString() } dbMangaId @@ -351,6 +356,7 @@ object BackupMangaHandler { this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt this[ChapterTable.version] = chapter.version + this[ChapterTable.memo] = chapter.memo.decodeToString() }.map { it[ChapterTable.id].value } } else { emptyList() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt index cf3941453..3818abea5 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt @@ -2,6 +2,7 @@ package suwayomi.tachidesk.manga.impl.backup.proto.models import kotlinx.serialization.Serializable import kotlinx.serialization.protobuf.ProtoNumber +import suwayomi.tachidesk.manga.impl.util.lang.JsonObjectEmptyBytes @Serializable data class BackupChapter( @@ -22,6 +23,7 @@ data class BackupChapter( // syncyomi @ProtoNumber(11) var lastModifiedAt: Long = 0, @ProtoNumber(12) var version: Long = 0, + @ProtoNumber(13) var memo: ByteArray = JsonObjectEmptyBytes, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt index 3dc29f640..49c65b88b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt @@ -3,6 +3,7 @@ package suwayomi.tachidesk.manga.impl.backup.proto.models import eu.kanade.tachiyomi.source.model.UpdateStrategy import kotlinx.serialization.Serializable import kotlinx.serialization.protobuf.ProtoNumber +import suwayomi.tachidesk.manga.impl.util.lang.JsonObjectEmptyBytes @Serializable data class BackupManga( @@ -37,6 +38,8 @@ data class BackupManga( // syncyomi @ProtoNumber(106) var lastModifiedAt: Long = 0, @ProtoNumber(109) var version: Long = 0, + @ProtoNumber(111) var initialized: Boolean = false, + @ProtoNumber(13) var memo: ByteArray = JsonObjectEmptyBytes, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt index b2a0379b0..4f06a2d31 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt @@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceFactory +import eu.kanade.tachiyomi.source.local.LocalSource import io.github.oshai.kotlinlogging.KotlinLogging import net.dongliu.apk.parser.ApkFile import net.dongliu.apk.parser.bean.Icon @@ -23,15 +24,17 @@ import okio.source import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.select import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.extension.ExtensionsList.extensionTableAsDataClass -import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi import suwayomi.tachidesk.manga.impl.util.PackageTools import suwayomi.tachidesk.manga.impl.util.PackageTools.EXTENSION_FEATURE import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN +import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_CONTENT_WARNING +import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_EXTENSION_LIB +import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_NAME import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_NSFW import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_SOURCE_CLASS import suwayomi.tachidesk.manga.impl.util.PackageTools.dex2jar @@ -62,18 +65,20 @@ object Extension { suspend fun installExtension(pkgName: String): Int { logger.debug { "Installing $pkgName" } - val extensionRecord = extensionTableAsDataClass().first { it.pkgName == pkgName } + val apkUrl = + transaction { + ExtensionTable + .select(ExtensionTable.apkUrl) + .where { ExtensionTable.pkgName eq pkgName } + .firstOrNull() + ?.get(ExtensionTable.apkUrl) + } ?: throw NullPointerException("Could not find extension $pkgName") return installAPK { - val apkURL = - ExtensionGithubApi.getApkUrl( - extensionRecord.repo ?: throw NullPointerException("Could not find extension repo"), - extensionRecord.apkName, - ) - val apkName = Uri.parse(apkURL).lastPathSegment!! + val apkName = Uri.parse(apkUrl).lastPathSegment!! val apkSavePath = "${applicationDirs.extensionsRoot}/$apkName" // download apk file - downloadAPKFile(apkURL, apkSavePath) + downloadAPKFile(apkUrl, apkSavePath) apkSavePath } @@ -148,7 +153,10 @@ object Extension { // throw Exception("This apk is not a signed with the official tachiyomi signature") // } - val isNsfw = packageInfo.applicationInfo.metaData.getString(METADATA_NSFW) == "1" + var contentWarning = packageInfo.applicationInfo.metaData.getInt(METADATA_CONTENT_WARNING) + if (contentWarning == 0) { + contentWarning = packageInfo.applicationInfo.metaData.getInt(METADATA_NSFW) + } val className = packageInfo.packageName + packageInfo.applicationInfo.metaData.getString(METADATA_SOURCE_CLASS) @@ -181,9 +189,16 @@ object Extension { } val extensionName = - packageInfo.applicationInfo.nonLocalizedLabel - .toString() - .substringAfter("Tachiyomi: ") + packageInfo.applicationInfo.metaData.getString(METADATA_NAME) + ?: packageInfo.applicationInfo.nonLocalizedLabel + .toString() + .substringAfter("Tachiyomi: ") + + val extensionLibVersion = + packageInfo.applicationInfo.metaData + .getString(METADATA_EXTENSION_LIB) + .takeUnless { it == "0" } + ?: packageInfo.versionName.substringBeforeLast('.') // update extension info transaction { @@ -193,9 +208,10 @@ object Extension { it[name] = extensionName it[this.pkgName] = packageInfo.packageName it[versionName] = packageInfo.versionName - it[versionCode] = packageInfo.versionCode + it[versionCode] = packageInfo.versionCode.toLong() + it[extensionLib] = extensionLibVersion it[lang] = extensionLang - it[this.isNsfw] = isNsfw + it[this.contentWarning] = contentWarning } } @@ -204,7 +220,7 @@ object Extension { it[this.isInstalled] = true it[this.classFQName] = className it[versionName] = packageInfo.versionName - it[versionCode] = packageInfo.versionCode + it[versionCode] = packageInfo.versionCode.toLong() } val extensionId = @@ -220,7 +236,7 @@ object Extension { it[name] = httpSource.name it[lang] = httpSource.lang it[extension] = extensionId - it[SourceTable.isNsfw] = isNsfw + it[this.contentWarning] = contentWarning } logger.debug { "Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}" } } @@ -343,7 +359,9 @@ object Extension { logger.debug { "Uninstalling $pkgName" } val extensionRecord = transaction { ExtensionTable.selectAll().where { ExtensionTable.pkgName eq pkgName }.first() } - val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk") + val fileNameWithoutType = + extensionRecord[ExtensionTable.apkName]?.substringBefore(".apk") + ?: throw NullPointerException("Missing $pkgName apkName") val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar" val sources = transaction { @@ -359,6 +377,7 @@ object Extension { ExtensionTable.update({ ExtensionTable.pkgName eq pkgName }) { it[isInstalled] = false it[hasUpdate] = false + it[apkName] = null } } @@ -385,8 +404,7 @@ object Extension { it[versionName] = targetExtension.versionName it[versionCode] = targetExtension.versionCode it[lang] = targetExtension.lang - it[isNsfw] = targetExtension.isNsfw - it[apkName] = targetExtension.apkName + it[contentWarning] = targetExtension.contentWarning.ordinal it[iconUrl] = targetExtension.iconUrl it[hasUpdate] = false } @@ -394,17 +412,21 @@ object Extension { return installExtension(pkgName) } - suspend fun getExtensionIcon(apkName: String): Pair { - val iconUrl = - if (apkName == "localSource") { - "" - } else { - transaction { ExtensionTable.selectAll().where { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl] - } - + suspend fun getExtensionIcon(pkgName: String): Pair { val cacheSaveDir = "${applicationDirs.extensionsRoot}/icon" - return getImageResponse(cacheSaveDir, apkName) { + if (pkgName == LocalSource::class.java.`package`.name) { + return getImageResponse(cacheSaveDir, "localSource") { + network.client + .newCall(GET("", cache = CacheControl.FORCE_NETWORK)) + .await() + } + } + + val iconUrl = + transaction { ExtensionTable.selectAll().where { ExtensionTable.pkgName eq pkgName }.first() }[ExtensionTable.iconUrl] + + return getImageResponse(cacheSaveDir, pkgName) { network.client .newCall( GET(iconUrl, cache = CacheControl.FORCE_NETWORK), @@ -412,5 +434,5 @@ object Extension { } } - fun getExtensionIconUrl(apkName: String): String = "/api/v1/extension/icon/$apkName" + fun proxyExtensionIconUrl(pkgName: String): String = "/api/v1/extension/icon/$pkgName" } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt new file mode 100644 index 000000000..7103ef407 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt @@ -0,0 +1,269 @@ +package suwayomi.tachidesk.manga.impl.extension + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.awaitSuccess +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.serialization.decodeFromByteArray +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.okio.decodeFromBufferedSource +import kotlinx.serialization.protobuf.ProtoBuf +import okio.BufferedSource +import okio.buffer +import okio.gzip +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import suwayomi.tachidesk.manga.impl.extension.github.NetworkExtensionStore +import suwayomi.tachidesk.manga.impl.extension.github.NetworkLegacyExtension +import suwayomi.tachidesk.manga.impl.extension.github.NetworkLegacyExtensionRepo +import suwayomi.tachidesk.manga.impl.extension.github.toExtensionInfo +import suwayomi.tachidesk.manga.impl.extension.github.toExtensionInfos +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.server.serverConfig +import uy.kohesive.injekt.injectLazy +import kotlin.coroutines.cancellation.CancellationException + +object ExtensionStoreService { + private val logger = KotlinLogging.logger {} + + val network: NetworkHelper by injectLazy() + val protoBuf: ProtoBuf by injectLazy() + val json: Json by injectLazy() + + suspend fun fetch(indexUrl: String): ExtensionStore { + var updatedIndexUrl: String = indexUrl + return try { + val response = network.client.newCall(GET(updatedIndexUrl)).awaitSuccess() + response.body.source().decompressIfGzipped().use { source -> + val networkStore = + when (source.peek().readByte()) { + // "[..." + 0x5B.toByte() -> { + run { + if (!indexUrl.endsWith("/index.min.json")) { + throw IllegalArgumentException("Provided legacy store url is not valid") + } + updatedIndexUrl = indexUrl.replace("/index.min.json", "/repo.json") + network.client.newCall(GET(updatedIndexUrl)).awaitSuccess().body.source().use { + json.decodeFromBufferedSource(it) + } + } + } + + // "{..." + 0x7B.toByte() -> { + try { + json.decodeFromBufferedSource(source.peek()) + } catch (_: IllegalArgumentException) { + json.decodeFromBufferedSource(source) + } + } + + else -> { + protoBuf.decodeFromByteArray(source.readByteArray()) + } + } + + if (networkStore is NetworkLegacyExtensionRepo && networkStore.indexV2 != null) { + return fetch(networkStore.indexV2) + } + + networkStore.toExtensionStore(updatedIndexUrl) + } + } catch (e: Exception) { + if (e is CancellationException) throw e + logger.error(e) { "Failed to fetch extension store '$indexUrl'" } + throw e + } + } + + fun upsert(store: ExtensionStore) { + transaction { + val existing = + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq store.indexUrl } + .firstOrNull() + + if (existing == null) { + ExtensionStoreTable.insert { + it[name] = store.name + it[badgeLabel] = store.badgeLabel + it[signingKey] = store.signingKey + it[contactWebsite] = store.contact.website + it[contactDiscord] = store.contact.discord + it[indexUrl] = store.indexUrl + it[isLegacy] = store.isLegacy + it[extensionListUrl] = store.extensionListUrl + } + } else { + ExtensionStoreTable.update({ ExtensionStoreTable.indexUrl eq store.indexUrl }) { + it[name] = store.name + it[badgeLabel] = store.badgeLabel + it[signingKey] = store.signingKey + it[contactWebsite] = store.contact.website + it[contactDiscord] = store.contact.discord + it[isLegacy] = store.isLegacy + it[extensionListUrl] = store.extensionListUrl + } + } + } + } + + suspend fun getAndRefresh(): List { + val stores = + transaction { + ExtensionStoreTable.selectAll().toList() + } + var needsPrefUpdate = false + val updateStores = + stores.mapNotNull { storeRow -> + val oldIndexUrl = storeRow[ExtensionStoreTable.indexUrl] + val oldName = storeRow[ExtensionStoreTable.name] + try { + val store = fetch(oldIndexUrl) + if (store.indexUrl != oldIndexUrl) { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl eq oldIndexUrl } + } + needsPrefUpdate = true + } + upsert(store) + store + } catch (e: Exception) { + logger.warn(e) { "Failed to fetch extension store '$oldName ($oldIndexUrl)'" } + null + } + } + if (needsPrefUpdate) syncDbToPrefs() + return updateStores + } + + fun syncDbToPrefs() { + val dbStores = + transaction { + ExtensionStoreTable + .selectAll() + .map { it[ExtensionStoreTable.indexUrl] } + .toSet() + } + + val currentPrefs = serverConfig.extensionStores.value.toSet() + val toAdd = dbStores - currentPrefs + val toRemove = currentPrefs - dbStores + + if (toAdd.isNotEmpty()) { + serverConfig.extensionStores.value = (serverConfig.extensionStores.value + toAdd).distinct() + } + + if (toRemove.isNotEmpty()) { + serverConfig.extensionStores.value = serverConfig.extensionStores.value.filterNot { it in toRemove } + } + } + + suspend fun syncPrefsToDb() { + val prefUrls = serverConfig.extensionStores.value.toSet() + + val dbStores = + transaction { + ExtensionStoreTable.selectAll().associateBy { it[ExtensionStoreTable.indexUrl] } + } + + val toAdd = prefUrls - dbStores.keys + val toRemove = (dbStores.keys - prefUrls).toMutableSet() + var needsPrefUpdate = toRemove.isNotEmpty() + + toAdd.forEach { url -> + try { + val store = fetch(url) + if (store.indexUrl != url) { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl eq url } + } + needsPrefUpdate = true + toRemove -= store.indexUrl + } + upsert(store) + } catch (e: Exception) { + logger.warn(e) { "Failed to sync preference store '$url' to database" } + } + } + + if (toRemove.isNotEmpty()) { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl inList toRemove.toList() } + } + } + if (needsPrefUpdate) { + syncDbToPrefs() + } + } + + suspend fun getExtensions(store: ExtensionStore): List { + val extensions = + if (store.extensionListUrl != null) { + val response = network.client.newCall(GET(store.extensionListUrl)).awaitSuccess() + response.body.source().decompressIfGzipped().use { source -> + when (source.peek().readByte()) { + // "{..." + 0x7B.toByte() -> { + json.decodeFromBufferedSource(source) + } + + else -> { + protoBuf.decodeFromByteArray( + source.readByteArray(), + ) + } + }.toExtensionInfos(store) + } + } else if (!store.isLegacy) { + val response = network.client.newCall(GET(store.indexUrl)).awaitSuccess() + response.body.source().decompressIfGzipped().use { source -> + when (source.peek().readByte()) { + // "{..." + 0x7B.toByte() -> json.decodeFromBufferedSource(source) + + else -> protoBuf.decodeFromByteArray(source.readByteArray()) + }.extensionList!! + .toExtensionInfos(store) + } + } else { + val storeBaseUrl = store.indexUrl.removeSuffix("/repo.json") + val response = network.client.newCall(GET("$storeBaseUrl/index.min.json")).awaitSuccess() + response.body.source().use { source -> + json + .decodeFromBufferedSource>(source) + .map { it.toExtensionInfo(store, storeBaseUrl) } + } + } + return extensions + } + + private fun BufferedSource.decompressIfGzipped(): BufferedSource { + val isGzip = + peek().use { peeked -> + try { + peeked.readShort().toInt() == 0x1f8b + } catch (_: Exception) { + false + } + } + + return if (isGzip) gzip().buffer() else this + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt index 0398f1f5f..fc4fbafdc 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt @@ -21,12 +21,11 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.extension.Extension.getExtensionIconUrl -import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi -import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension +import suwayomi.tachidesk.manga.impl.extension.Extension.proxyExtensionIconUrl +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo import suwayomi.tachidesk.manga.model.table.ExtensionTable -import suwayomi.tachidesk.server.serverConfig import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration.Companion.seconds @@ -34,23 +33,23 @@ object ExtensionsList { private val logger = KotlinLogging.logger {} var lastUpdateCheck: Long = 0 - var updateMap = ConcurrentHashMap() + var updateMap = ConcurrentHashMap() suspend fun fetchExtensions() { - // update if 60 seconds has passed or requested offline and database is empty - val extensions = - serverConfig.extensionRepos.value.map { repo -> - kotlin - .runCatching { - ExtensionGithubApi.findExtensions(repo.repoUrlReplace()) - }.onFailure { - logger.warn(it) { - "Failed to fetch extensions for repo: $repo" - } - } + val allExtensions = mutableListOf() + + ExtensionStoreService.getAndRefresh().forEach { store -> + try { + val extensions = ExtensionStoreService.getExtensions(store) + allExtensions.addAll(extensions) + } catch (e: Exception) { + logger.warn(e) { + "Failed to fetch extensions for store: ${store.indexUrl}" + } } - val foundExtensions = extensions.mapNotNull { it.getOrNull() }.flatten() - updateExtensionDatabase(foundExtensions) + } + + updateExtensionDatabase(allExtensions) } suspend fun fetchExtensionsCached() { @@ -74,25 +73,25 @@ object ExtensionsList { transaction { ExtensionTable.selectAll().filter { it[ExtensionTable.name] != LocalSource.EXTENSION_NAME }.map { ExtensionDataClass( - it[ExtensionTable.repo], - it[ExtensionTable.apkName], - getExtensionIconUrl(it[ExtensionTable.apkName]), - it[ExtensionTable.name], - it[ExtensionTable.pkgName], - it[ExtensionTable.versionName], - it[ExtensionTable.versionCode], - it[ExtensionTable.lang], - it[ExtensionTable.isNsfw], - it[ExtensionTable.isInstalled], - it[ExtensionTable.hasUpdate], - it[ExtensionTable.isObsolete], + repo = it[ExtensionTable.storeIndexUrl], + apkName = it[ExtensionTable.apkName].orEmpty(), + iconUrl = proxyExtensionIconUrl(it[ExtensionTable.pkgName]), + name = it[ExtensionTable.name], + pkgName = it[ExtensionTable.pkgName], + versionName = it[ExtensionTable.versionName], + versionCode = it[ExtensionTable.versionCode].toInt(), + lang = it[ExtensionTable.lang], + isNsfw = it[ExtensionTable.contentWarning] >= ContentWarning.MIXED.ordinal, + installed = it[ExtensionTable.isInstalled], + hasUpdate = it[ExtensionTable.hasUpdate], + obsolete = it[ExtensionTable.isObsolete], ) } } private val updateExtensionDatabaseMutex = Mutex() - private suspend fun updateExtensionDatabase(foundExtensions: List) { + private suspend fun updateExtensionDatabase(foundExtensions: List) { updateExtensionDatabaseMutex.withLock { transaction { val uniqueExtensions = @@ -106,10 +105,10 @@ object ExtensionsList { .selectAll() .toList() .associateBy { it[ExtensionTable.pkgName] } - val extensionsToUpdate = mutableListOf>() - val extensionsToInsert = mutableListOf() + val extensionsToUpdate = mutableListOf>() + val extensionsToInsert = mutableListOf() val extensionsToDelete = - installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) -> + installedExtensions.filter { it.value[ExtensionTable.storeIndexUrl] != null }.mapNotNull { (pkgName, extension) -> extension.takeUnless { uniqueExtensions.any { it.pkgName == pkgName } } } uniqueExtensions.forEach { @@ -132,7 +131,7 @@ object ExtensionsList { addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) // Always update icon url and repo this[ExtensionTable.iconUrl] = foundExtension.iconUrl - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl // add these because batch updates need matching columns this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] @@ -168,13 +167,14 @@ object ExtensionsList { extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) -> addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) // extension is not installed, so we can overwrite the data without a care - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.extensionLib] = foundExtension.extensionLib this[ExtensionTable.versionName] = foundExtension.versionName this[ExtensionTable.versionCode] = foundExtension.versionCode this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.contentWarning] = foundExtension.contentWarning.ordinal + this[ExtensionTable.apkUrl] = foundExtension.apkUrl this[ExtensionTable.iconUrl] = foundExtension.iconUrl } }.toExecutable() @@ -183,14 +183,15 @@ object ExtensionsList { } if (extensionsToInsert.isNotEmpty()) { ExtensionTable.batchInsert(extensionsToInsert) { foundExtension -> - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl this[ExtensionTable.name] = foundExtension.name this[ExtensionTable.pkgName] = foundExtension.pkgName + this[ExtensionTable.extensionLib] = foundExtension.extensionLib this[ExtensionTable.versionName] = foundExtension.versionName this[ExtensionTable.versionCode] = foundExtension.versionCode this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.contentWarning] = foundExtension.contentWarning.ordinal + this[ExtensionTable.apkUrl] = foundExtension.apkUrl this[ExtensionTable.iconUrl] = foundExtension.iconUrl } } @@ -215,16 +216,4 @@ object ExtensionsList { } } } - - private fun String.repoUrlReplace(): String = - if (contains("github")) { - replace(repoMatchRegex) { - "https://raw.githubusercontent.com/${it.groupValues[2]}/${it.groupValues[3]}/" + - (it.groupValues.getOrNull(4)?.ifBlank { null } ?: "repo") + - "/" + - (it.groupValues.getOrNull(5)?.ifBlank { null } ?: "index.min.json") - } - } else { - this - } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt new file mode 100644 index 000000000..68d485216 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt @@ -0,0 +1,14 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +interface BaseNetworkExtensionStore { + fun toExtensionStore(indexUrl: String): ExtensionStore +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt deleted file mode 100644 index 7da708a1f..000000000 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt +++ /dev/null @@ -1,107 +0,0 @@ -package suwayomi.tachidesk.manga.impl.extension.github - -/* - * Copyright (C) Contributors to the Suwayomi project - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.NetworkHelper -import eu.kanade.tachiyomi.network.awaitSuccess -import eu.kanade.tachiyomi.network.parseAs -import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json -import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX -import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN -import uy.kohesive.injekt.injectLazy - -object ExtensionGithubApi { - private val logger = KotlinLogging.logger {} - private val json: Json by injectLazy() - - @Serializable - private data class ExtensionJsonObject( - val name: String, - val pkg: String, - val apk: String, - val lang: String, - val code: Int, - val version: String, - val nsfw: Int, - val hasReadme: Int = 0, - val hasChangelog: Int = 0, - val sources: List?, - ) - - @Serializable - private data class ExtensionSourceJsonObject( - val name: String, - val lang: String, - val id: Long, - val baseUrl: String, - ) - - suspend fun findExtensions(repo: String): List { - val response = - client.newCall(GET(repo)).awaitSuccess() - - return with(json) { - response - .parseAs>() - .toExtensions(repo.substringBeforeLast('/') + '/') - } - } - - fun getApkUrl( - repo: String, - apkName: String, - ): String = "${repo}apk/$apkName" - - private val client by lazy { - val network: NetworkHelper by injectLazy() - network.client - .newBuilder() - .addNetworkInterceptor { chain -> - val originalResponse = chain.proceed(chain.request()) - originalResponse - .newBuilder() - .header("Content-Type", "application/json") - .build() - }.build() - } - - private fun List.toExtensions(repo: String): List = - this - .filter { - val libVersion = it.version.substringBeforeLast('.').toDouble() - libVersion in LIB_VERSION_MIN..LIB_VERSION_MAX - }.map { - OnlineExtension( - repo = repo, - name = it.name.substringAfter("Tachiyomi: "), - pkgName = it.pkg, - versionName = it.version, - versionCode = it.code, - lang = it.lang, - isNsfw = it.nsfw == 1, - hasReadme = it.hasReadme == 1, - hasChangelog = it.hasChangelog == 1, - sources = it.sources?.toExtensionSources() ?: emptyList(), - apkName = it.apk, - iconUrl = "${repo}icon/${it.pkg}.png", - ) - } - - private fun List.toExtensionSources(): List = - this.map { - OnlineExtensionSource( - name = it.name, - lang = it.lang, - id = it.id, - baseUrl = it.baseUrl, - ) - } -} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt new file mode 100644 index 000000000..1db92197b --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt @@ -0,0 +1,148 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonNames +import kotlinx.serialization.protobuf.ProtoNumber +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionSource +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@OptIn(ExperimentalSerializationApi::class) +@Serializable +data class NetworkExtensionStore( + @ProtoNumber(1) val name: String, + @ProtoNumber(2) val badgeLabel: String, + @ProtoNumber(3) val signingKey: String, + @ProtoNumber(4) val contact: Contact, + @ProtoNumber(101) val extensionList: ExtensionList?, + @ProtoNumber(102) val extensionListUrl: String?, +) : BaseNetworkExtensionStore { + @Serializable + data class Contact( + @ProtoNumber(1) val website: String, + @ProtoNumber(2) val discord: String?, + ) + + @Serializable + data class ExtensionList( + @ProtoNumber(1) val extensions: List, + ) + + @Serializable + data class Extension( + @ProtoNumber(1) val name: String, + @ProtoNumber(2) val packageName: String, + @ProtoNumber(3) val resources: Resources, + @ProtoNumber(4) val extensionLib: String, + @ProtoNumber(5) val versionCode: Long, + @ProtoNumber(6) val versionName: String, + @ProtoNumber(7) val contentWarning: ContentWarning, + @ProtoNumber(8) val sources: List, + ) + + @Serializable + data class Resources( + @ProtoNumber(1) val apkUrl: String, + @ProtoNumber(2) val iconUrl: String, + ) + + @Serializable + data class Source( + @ProtoNumber(1) val id: Long, + @ProtoNumber(2) val name: String, + @ProtoNumber(3) val language: String, + @ProtoNumber(4) val homeUrl: String = "", + @ProtoNumber(5) val mirrorUrls: List = emptyList(), + // @ProtoNumber(6) val contentWarning: ContentWarning = ContentWarning.SAFE, + @ProtoNumber(7) val message: String? = null, + ) + + @Serializable + enum class ContentWarning { + @ProtoNumber(0) + @JsonNames("CONTENT_WARNING_UNSPECIFIED") + UNSPECIFIED, + + @ProtoNumber(1) + @JsonNames("CONTENT_WARNING_SAFE") + SAFE, + + @ProtoNumber(2) + @JsonNames("CONTENT_WARNING_MIXED") + MIXED, + + @ProtoNumber(3) + @JsonNames("CONTENT_WARNING_NSFW") + NSFW, + } + + override fun toExtensionStore(indexUrl: String): ExtensionStore = + ExtensionStore( + indexUrl = indexUrl, + name = name, + badgeLabel = badgeLabel, + signingKey = signingKey, + contact = + ExtensionStore.Contact( + website = contact.website, + discord = contact.discord, + ), + isLegacy = false, + extensionListUrl = extensionListUrl, + ) +} + +fun NetworkExtensionStore.ExtensionList.toExtensionInfos(store: ExtensionStore): List = + extensions.map { extension -> + val lang = extension.sources.map { it.language }.toSet() + ExtensionInfo( + storeIndexUrl = store.indexUrl, + name = extension.name, + pkgName = extension.packageName, + apkUrl = extension.resources.apkUrl, + iconUrl = extension.resources.iconUrl, + extensionLib = extension.extensionLib, + versionCode = extension.versionCode, + versionName = extension.versionName, + lang = if (lang.size == 1) lang.first() else "all", + contentWarning = + when (extension.contentWarning) { + NetworkExtensionStore.ContentWarning.SAFE, + NetworkExtensionStore.ContentWarning.UNSPECIFIED, + -> ContentWarning.SAFE + + NetworkExtensionStore.ContentWarning.MIXED -> ContentWarning.MIXED + + NetworkExtensionStore.ContentWarning.NSFW -> ContentWarning.NSFW + }, + sources = + extension.sources.map { source -> + ExtensionSource( + id = source.id, + name = source.name, + lang = source.language, + homeUrl = source.homeUrl, + message = source.message, + contentWarning = + when (extension.contentWarning) { // todo source.contentWarning + NetworkExtensionStore.ContentWarning.SAFE, + NetworkExtensionStore.ContentWarning.UNSPECIFIED, + -> ContentWarning.SAFE + + NetworkExtensionStore.ContentWarning.MIXED -> ContentWarning.MIXED + + NetworkExtensionStore.ContentWarning.NSFW -> ContentWarning.NSFW + }, + ) + }, + ) + } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt new file mode 100644 index 000000000..bffa2bc66 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt @@ -0,0 +1,77 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionSource +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@OptIn(ExperimentalSerializationApi::class) +@Serializable +data class NetworkLegacyExtension( + val name: String, + val pkg: String, + val apk: String, + val lang: String, + val version: String, + val code: Long, + val nsfw: Int, + val sources: List? = null, +) { + @Serializable + data class Source( + val id: Long, + val lang: String, + val name: String, + val baseUrl: String, + ) +} + +fun NetworkLegacyExtension.toExtensionInfo( + store: ExtensionStore, + storeBaseUrl: String, +): ExtensionInfo = + ExtensionInfo( + storeIndexUrl = store.indexUrl, + name = name.substringAfter("Tachiyomi: "), + pkgName = pkg, + apkUrl = "$storeBaseUrl/apk/$apk", + iconUrl = "$storeBaseUrl/icon/$pkg.png", + extensionLib = version.substringBeforeLast('.'), + versionCode = code, + versionName = version, + lang = lang, + contentWarning = if (nsfw == 1) ContentWarning.MIXED else ContentWarning.SAFE, + sources = + if (sources.isNullOrEmpty()) { + listOf( + ExtensionSource( + id = 0, + name = name, + lang = lang, + homeUrl = "", + message = null, + contentWarning = if (nsfw == 1) ContentWarning.MIXED else ContentWarning.SAFE, + ), + ) + } else { + sources.map { source -> + ExtensionSource( + id = source.id, + name = source.name, + lang = source.lang, + homeUrl = source.baseUrl, + message = null, + contentWarning = if (nsfw == 1) ContentWarning.MIXED else ContentWarning.SAFE, + ) + } + }, + ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt new file mode 100644 index 000000000..1963e1ce2 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt @@ -0,0 +1,41 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@Serializable +data class NetworkLegacyExtensionRepo( + @SerialName("index_v2") val indexV2: String?, + val meta: Meta, +) : BaseNetworkExtensionStore { + @Serializable + data class Meta( + val name: String, + val shortName: String?, + val website: String, + val signingKeyFingerprint: String, + ) + + override fun toExtensionStore(indexUrl: String): ExtensionStore = + ExtensionStore( + indexUrl = indexUrl, + name = meta.name, + badgeLabel = meta.shortName ?: meta.name, + signingKey = meta.signingKeyFingerprint, + contact = + ExtensionStore.Contact( + website = meta.website, + discord = null, + ), + isLegacy = true, + extensionListUrl = null, + ) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt deleted file mode 100644 index 8a399b647..000000000 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt +++ /dev/null @@ -1,30 +0,0 @@ -package suwayomi.tachidesk.manga.impl.extension.github - -/* - * Copyright (C) Contributors to the Suwayomi project - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -data class OnlineExtensionSource( - val name: String, - val lang: String, - val id: Long, - val baseUrl: String, -) - -data class OnlineExtension( - val repo: String, - val name: String, - val pkgName: String, - val apkName: String, - val lang: String, - val versionCode: Int, - val versionName: String, - val isNsfw: Boolean, - val hasReadme: Boolean, - val hasChangelog: Boolean, - val sources: List, - val iconUrl: String, -) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt index 1e7b34ca9..fe4d61a2d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt @@ -31,7 +31,6 @@ import kotlinx.coroutines.sync.withPermit import suwayomi.tachidesk.global.impl.sync.SyncManager import suwayomi.tachidesk.manga.impl.Category import suwayomi.tachidesk.manga.impl.CategoryManga -import suwayomi.tachidesk.manga.impl.Chapter import suwayomi.tachidesk.manga.impl.Manga import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude @@ -311,10 +310,10 @@ class Updater : IUpdater { tracker[job.manga.id] = try { logger.info { "Updating ${job.manga}" } - if (serverConfig.updateMangas.value || !job.manga.initialized) { - Manga.getManga(job.manga.id, true) - } - Chapter.getChapterList(job.manga.id, true) + Manga.updateMangaAndChapters( + job.manga.id, + updateManga = serverConfig.updateMangas.value || !job.manga.initialized, + ) job.copy(status = JobStatus.COMPLETE) } catch (e: Exception) { logger.error(e) { "Error while updating ${job.manga}" } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt index 9194d3244..1671be0a7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt @@ -40,8 +40,13 @@ object PackageTools { const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class" const val METADATA_SOURCE_FACTORY = "tachiyomi.extension.factory" const val METADATA_NSFW = "tachiyomi.extension.nsfw" + + const val METADATA_NAME = "tachiyomix.name" + const val METADATA_EXTENSION_LIB = "tachiyomix.extensionLib" + const val METADATA_CONTENT_WARNING = "tachiyomix.contentWarning" + const val LIB_VERSION_MIN = 1.3 - const val LIB_VERSION_MAX = 1.5 + const val LIB_VERSION_MAX = 1.6 /** * Convert dex to jar, a wrapper for the dex2jar library diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/lang/JsonObject.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/lang/JsonObject.kt new file mode 100644 index 000000000..c6f2b67db --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/lang/JsonObject.kt @@ -0,0 +1,10 @@ +package suwayomi.tachidesk.manga.impl.util.lang + +import kotlinx.serialization.json.JsonObject + +val JsonObjectEmpty = JsonObject(emptyMap()) + +val JsonObjectEmptyBytes = byteArrayOf(0x7B, 0x7D) + +val JsonObject.Companion.EMPTY: JsonObject + inline get() = JsonObjectEmpty diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt index 99952debf..2de64c464 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt @@ -45,7 +45,9 @@ object GetCatalogueSource { ExtensionTable.selectAll().where { ExtensionTable.id eq extensionId }.first() } - val apkName = extensionRecord[ExtensionTable.apkName] + val apkName = + extensionRecord[ExtensionTable.apkName] + ?: throw NullPointerException("Missing apkName") val className = extensionRecord[ExtensionTable.classFQName] val jarName = apkName.substringBefore(".apk") + ".jar" val jarPath = "${applicationDirs.extensionsRoot}/$jarName" diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt index a7a5a3641..372bfc230 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt @@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.SMangaUpdate import rx.Observable open class StubSource( @@ -23,9 +24,17 @@ open class StubSource( override val name: String get() = id.toString() + override suspend fun getPopularManga(page: Int): MangasPage = throw getSourceNotInstalledException() + @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPopularManga")) override fun fetchPopularManga(page: Int): Observable = Observable.error(getSourceNotInstalledException()) + override suspend fun getSearchManga( + page: Int, + query: String, + filters: FilterList, + ): MangasPage = throw getSourceNotInstalledException() + @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getSearchManga")) override fun fetchSearchManga( page: Int, @@ -33,17 +42,28 @@ open class StubSource( filters: FilterList, ): Observable = Observable.error(getSourceNotInstalledException()) + override suspend fun getLatestUpdates(page: Int): MangasPage = throw getSourceNotInstalledException() + @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getLatestUpdates")) override fun fetchLatestUpdates(page: Int): Observable = Observable.error(getSourceNotInstalledException()) override fun getFilterList(): FilterList = FilterList() + override suspend fun getMangaUpdate( + manga: SManga, + chapters: List, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate = throw getSourceNotInstalledException() + @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getMangaDetails")) override fun fetchMangaDetails(manga: SManga): Observable = Observable.error(getSourceNotInstalledException()) @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getChapterList")) override fun fetchChapterList(manga: SManga): Observable> = Observable.error(getSourceNotInstalledException()) + override suspend fun getPageList(chapter: SChapter): List = throw getSourceNotInstalledException() + @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPageList")) override fun fetchPageList(chapter: SChapter): Observable> = Observable.error(getSourceNotInstalledException()) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt index ccb430fa1..60a4b5a72 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt @@ -46,7 +46,7 @@ object ImageResponse { /** * Get a cached image response * - * Note: The caller should also call [clearCachedImage] when appropriate + * Note: The caller should also call [ImageResponse.clearCachedImage] when appropriate * * @param cacheSavePath where to save the cached image. Caller should decide to use perma cache or temp cache (OS temp dir) * @param fileName what the saved cache file should be named diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt index 918114774..31c9e2dfd 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt @@ -1,10 +1,13 @@ package suwayomi.tachidesk.manga.model.dataclass +import com.fasterxml.jackson.annotation.JsonIgnore import eu.kanade.tachiyomi.source.model.SChapter +import kotlinx.serialization.json.JsonObject import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.manga.impl.Chapter.getChapterMetaMap +import suwayomi.tachidesk.manga.impl.util.lang.EMPTY import suwayomi.tachidesk.manga.model.table.ChapterTable /* @@ -43,6 +46,8 @@ data class ChapterDataClass( val pageCount: Int = -1, val lastModifiedAt: Long = 0, val version: Long = 0, + @JsonIgnore + val memo: JsonObject = JsonObject.EMPTY, ) { companion object { fun fromSChapter( @@ -60,6 +65,7 @@ data class ChapterDataClass( uploadDate = sChapter.date_upload, chapterNumber = sChapter.chapter_number, scanlator = sChapter.scanlator, + memo = sChapter.memo, index = index, fetchedAt = fetchedAt, realUrl = realUrl, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt new file mode 100644 index 000000000..22c0b1b17 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt @@ -0,0 +1,42 @@ +package suwayomi.tachidesk.manga.model.dataclass + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +data class ExtensionInfo( + val storeIndexUrl: String, + val name: String, + val pkgName: String, + val apkUrl: String, + val iconUrl: String, + val extensionLib: String, + val versionCode: Long, + val versionName: String, + val lang: String, + val contentWarning: ContentWarning, + val sources: List, +) + +data class ExtensionSource( + val id: Long, + val name: String, + val lang: String, + val homeUrl: String, + val message: String?, + val contentWarning: ContentWarning, +) + +enum class ContentWarning { + SAFE, + MIXED, + NSFW, + ; + + companion object { + fun valueOf(contentWarning: Int) = entries.find { it.ordinal == contentWarning } ?: SAFE + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt new file mode 100644 index 000000000..4fad0b5d6 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt @@ -0,0 +1,23 @@ +package suwayomi.tachidesk.manga.model.dataclass + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +data class ExtensionStore( + val indexUrl: String, + val name: String, + val badgeLabel: String, + val signingKey: String, + val contact: Contact, + val isLegacy: Boolean, + val extensionListUrl: String?, +) { + data class Contact( + val website: String, + val discord: String?, + ) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt index ecf882554..1d49bf1d3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt @@ -7,8 +7,11 @@ package suwayomi.tachidesk.manga.model.dataclass * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import com.fasterxml.jackson.annotation.JsonIgnore import eu.kanade.tachiyomi.source.model.UpdateStrategy +import kotlinx.serialization.json.JsonObject import suwayomi.tachidesk.manga.impl.Manga.getMangaMetaMap +import suwayomi.tachidesk.manga.impl.util.lang.EMPTY import suwayomi.tachidesk.manga.impl.util.lang.trimAll import suwayomi.tachidesk.manga.model.table.MangaStatus import java.time.Instant @@ -44,6 +47,8 @@ data class MangaDataClass( val trackers: List? = null, val lastModifiedAt: Long = 0, val version: Long = 0, + @JsonIgnore + val memo: JsonObject = JsonObject.EMPTY, ) { override fun toString(): String = "\"$title\" (id= $id) (sourceId= $sourceId)" diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt index a672d906a..c37d0180d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt @@ -7,11 +7,13 @@ package suwayomi.tachidesk.manga.model.table * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ReferenceOption import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.dao.id.IntIdTable import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar +import suwayomi.tachidesk.manga.model.table.columns.unlimitedVarchar object ChapterTable : IntIdTable() { val url = varchar("url", 2048) @@ -42,6 +44,8 @@ object ChapterTable : IntIdTable() { val lastModifiedAt = long("last_modified_at").default(0) val version = long("version").default(0) val isSyncing = bool("is_syncing").default(false) + + val memo = unlimitedVarchar("memo") } fun ChapterTable.toDataClass(chapterEntry: ResultRow) = @@ -64,4 +68,5 @@ fun ChapterTable.toDataClass(chapterEntry: ResultRow) = pageCount = chapterEntry[pageCount], lastModifiedAt = chapterEntry[lastModifiedAt], version = chapterEntry[version], + memo = Json.decodeFromString(chapterEntry[memo]), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt new file mode 100644 index 000000000..84de416f5 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt @@ -0,0 +1,21 @@ +package suwayomi.tachidesk.manga.model.table + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.v1.core.dao.id.IntIdTable + +object ExtensionStoreTable : IntIdTable() { + val indexUrl = varchar("index_url", 2048).uniqueIndex() + val name = varchar("name", 256) + val badgeLabel = varchar("badge_label", 32) + val signingKey = varchar("signing_key", 512) + val contactWebsite = varchar("contact_website", 2048) + val contactDiscord = varchar("contact_discord", 2048).nullable() + val isLegacy = bool("is_legacy").default(false) + val extensionListUrl = varchar("extension_list_url", 2048).nullable() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt index b73cfef24..36d3e2bbe 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt @@ -10,8 +10,8 @@ package suwayomi.tachidesk.manga.model.table import org.jetbrains.exposed.v1.core.dao.id.IntIdTable object ExtensionTable : IntIdTable() { - val apkName = varchar("apk_name", 1024) - val repo = varchar("repo", 1024).nullable() + val apkName = varchar("apk_name", 1024).nullable() + val storeIndexUrl = varchar("store_index_url", 2048).nullable().index() // default is the local source icon from tachiyomi @Suppress("ktlint:standard:max-line-length") @@ -23,10 +23,12 @@ object ExtensionTable : IntIdTable() { val name = varchar("name", 128) val pkgName = varchar("pkg_name", 128) + val apkUrl = varchar("apk_url", 2048) + val extensionLib = varchar("extension_lib", 16).nullable() val versionName = varchar("version_name", 16) - val versionCode = integer("version_code") + val versionCode = long("version_code") val lang = varchar("lang", 32) - val isNsfw = bool("is_nsfw") + val contentWarning = integer("content_warning") val isInstalled = bool("is_installed").default(false) val hasUpdate = bool("has_update").default(false) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt index 323d8142e..d4087c841 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt @@ -9,6 +9,7 @@ package suwayomi.tachidesk.manga.model.table import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.UpdateStrategy +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.dao.id.IntIdTable import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl @@ -48,6 +49,7 @@ object MangaTable : IntIdTable() { val lastModifiedAt = long("last_modified_at").default(0) val version = long("version").default(0) val isSyncing = bool("is_syncing").default(false) + val memo = unlimitedVarchar("memo") } fun MangaTable.toDataClass(mangaEntry: ResultRow) = @@ -72,6 +74,7 @@ fun MangaTable.toDataClass(mangaEntry: ResultRow) = updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]), lastModifiedAt = mangaEntry[lastModifiedAt], version = mangaEntry[version], + memo = Json.decodeFromString(mangaEntry[memo]), ) enum class MangaStatus( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt index bdbaa5479..09e77a1c3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt @@ -14,5 +14,5 @@ object SourceTable : IdTable() { val name = varchar("name", 128) val lang = varchar("lang", 32) val extension = reference("extension", ExtensionTable) - val isNsfw = bool("is_nsfw").default(false) + val contentWarning = integer("content_warning").default(0) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/opds/impl/OpdsFeedBuilder.kt b/server/src/main/kotlin/suwayomi/tachidesk/opds/impl/OpdsFeedBuilder.kt index 41c505190..3e5830528 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/opds/impl/OpdsFeedBuilder.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/opds/impl/OpdsFeedBuilder.kt @@ -3,6 +3,7 @@ package suwayomi.tachidesk.opds.impl import io.github.oshai.kotlinlogging.KotlinLogging import org.jetbrains.exposed.v1.core.SortOrder import suwayomi.tachidesk.i18n.MR +import suwayomi.tachidesk.manga.impl.Manga import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.opds.constants.OpdsConstants @@ -17,7 +18,6 @@ import suwayomi.tachidesk.opds.repository.ChapterRepository import suwayomi.tachidesk.opds.repository.MangaRepository import suwayomi.tachidesk.opds.repository.NavigationRepository import suwayomi.tachidesk.opds.util.OpdsDateUtil -import suwayomi.tachidesk.opds.util.OpdsStringUtil import suwayomi.tachidesk.opds.util.OpdsXmlUtil import suwayomi.tachidesk.server.serverConfig import java.util.Locale @@ -657,8 +657,7 @@ object OpdsFeedBuilder { // If no chapters are found in the database, attempt to fetch them from the source. if (chapterEntries.isEmpty() && totalChapters == 0L) { try { - suwayomi.tachidesk.manga.impl.Chapter - .fetchChapterList(mangaId) + Manga.updateMangaAndChapters(mangaId, updateManga = false) // Re-query after fetching. val (refetchedChapters, refetchedTotal) = diff --git a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/MangaRepository.kt b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/MangaRepository.kt index a5ae49560..e3656f1ff 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/MangaRepository.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/MangaRepository.kt @@ -9,7 +9,6 @@ import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.alias import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq -import org.jetbrains.exposed.v1.core.greater import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.inSubQuery import org.jetbrains.exposed.v1.core.intLiteral diff --git a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt index 97eb08f73..3a1ee62ee 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt @@ -4,7 +4,6 @@ import dev.icerock.moko.resources.StringResource import org.jetbrains.exposed.v1.core.JoinType import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.alias -import org.jetbrains.exposed.v1.core.count import org.jetbrains.exposed.v1.core.countDistinct import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.select @@ -138,9 +137,9 @@ object NavigationRepository { val query = SourceTable .join(ExtensionTable, JoinType.LEFT, onColumn = SourceTable.extension, otherColumn = ExtensionTable.id) - .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .where { ExtensionTable.isInstalled eq true } - .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .orderBy(SourceTable.name to SortOrder.ASC) val totalCount = query.count() @@ -152,7 +151,7 @@ object NavigationRepository { OpdsSourceNavEntry( id = it[SourceTable.id].value, name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]), - iconUrl = it[ExtensionTable.apkName].let { apkName -> Extension.getExtensionIconUrl(apkName) }, + iconUrl = it[ExtensionTable.pkgName].let { pkgName -> Extension.proxyExtensionIconUrl(pkgName) }, mangaCount = null, ) } @@ -177,13 +176,13 @@ object NavigationRepository { val query = baseJoin - .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName, mangaCount) + .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName, mangaCount) .where { MangaTable.inLibrary eq true } query.applyOpdsMangaFilter(activeFilters, excludeField = "source_id") query - .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .orderBy(SourceTable.name to SortOrder.ASC) val totalCount = query.count() @@ -199,7 +198,7 @@ object NavigationRepository { OpdsSourceNavEntry( id = it[SourceTable.id].value, name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]), - iconUrl = it[ExtensionTable.apkName].let { apkName -> Extension.getExtensionIconUrl(apkName) }, + iconUrl = it[ExtensionTable.pkgName].let { pkgName -> Extension.proxyExtensionIconUrl(pkgName) }, mangaCount = it[mangaCount], ) } @@ -210,12 +209,12 @@ object NavigationRepository { transaction { SourceTable .join(ExtensionTable, JoinType.LEFT, onColumn = SourceTable.extension, otherColumn = ExtensionTable.id) - .select(SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .select(SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .where { SourceTable.id eq sourceId } .firstOrNull() ?.let { val name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]) - val icon = Extension.getExtensionIconUrl(it[ExtensionTable.apkName]) + val icon = Extension.proxyExtensionIconUrl(it[ExtensionTable.pkgName]) Pair(name, icon) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 41b15cec3..ea77725a4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -41,6 +41,7 @@ import suwayomi.tachidesk.graphql.types.DatabaseType import suwayomi.tachidesk.i18n.LocalizationHelper import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport import suwayomi.tachidesk.manga.impl.download.DownloadManager +import suwayomi.tachidesk.manga.impl.extension.ExtensionStoreService import suwayomi.tachidesk.manga.impl.update.IUpdater import suwayomi.tachidesk.manga.impl.update.Updater import suwayomi.tachidesk.manga.impl.util.lang.renameTo @@ -519,4 +520,12 @@ fun applicationSetup() { GlobalScope.launch { CEFManager.init() } + + serverConfig.subscribeTo( + serverConfig.extensionStores, + { _ -> + ExtensionStoreService.syncPrefsToDb() + }, + ignoreInitialValue = false, + ) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt index 2891274cd..07c70cfb6 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt @@ -12,6 +12,7 @@ import com.zaxxer.hikari.HikariDataSource import de.neonew.exposed.migrations.loadMigrationsFrom import de.neonew.exposed.migrations.runMigrations import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.DatabaseConfig import org.jetbrains.exposed.v1.core.ExperimentalKeywordApi import org.jetbrains.exposed.v1.core.Schema @@ -140,6 +141,8 @@ object DBManager { "Idle: ${ds.hikariPoolMXBean.idleConnections}, " + "Waiting: ${ds.hikariPoolMXBean.threadsAwaitingConnection}" } + + val format = Json { prettyPrint = false } } private val logger = KotlinLogging.logger {} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt new file mode 100644 index 000000000..eef3275c0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt @@ -0,0 +1,89 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.SQLMigration +import suwayomi.tachidesk.graphql.types.DatabaseType +import suwayomi.tachidesk.manga.model.dataclass.ContentWarning +import suwayomi.tachidesk.server.database.migration.helpers.MAYBE_TYPE_PREFIX +import suwayomi.tachidesk.server.database.migration.helpers.UNLIMITED_TEXT +import suwayomi.tachidesk.server.database.migration.helpers.toSqlName +import suwayomi.tachidesk.server.serverConfig + +@Suppress("ClassName", "unused") +class M0057_AddNewExtensionApiFields : SQLMigration() { + fun postgresRename(): String = + "ALTER TABLE EXTENSION RENAME COLUMN " + "repo".toSqlName() + " TO " + "store_index_url".toSqlName() + ";" + + fun h2Rename(): String = + "ALTER TABLE EXTENSION ALTER COLUMN " + "repo".toSqlName() + " RENAME TO " + "store_index_url".toSqlName() + ";" + + override val sql by lazy { + """ + ALTER TABLE manga ADD COLUMN memo $UNLIMITED_TEXT DEFAULT '{}' NOT NULL; + ALTER TABLE chapter ADD COLUMN memo $UNLIMITED_TEXT DEFAULT '{}' NOT NULL; + ${ + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> postgresRename() + DatabaseType.H2 -> h2Rename() + } + } + ALTER TABLE EXTENSION ALTER COLUMN store_index_url ${MAYBE_TYPE_PREFIX}VARCHAR(2048); + CREATE INDEX extension_store_index_url ON EXTENSION (store_index_url); + ALTER TABLE EXTENSION ALTER COLUMN version_code ${MAYBE_TYPE_PREFIX}BIGINT; + ALTER TABLE EXTENSION ALTER COLUMN apk_name DROP NOT NULL; + ${ + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> postgresBackfill() + DatabaseType.H2 -> h2Backfill() + } + } + ALTER TABLE EXTENSION ADD COLUMN apk_url VARCHAR(2048); + ALTER TABLE EXTENSION ADD COLUMN content_warning INTEGER DEFAULT 0; + UPDATE EXTENSION SET content_warning = ${ContentWarning.MIXED.ordinal} WHERE is_nsfw = TRUE; + ALTER TABLE EXTENSION DROP COLUMN is_nsfw; + ALTER TABLE SOURCE ADD COLUMN content_warning INTEGER DEFAULT 0; + UPDATE SOURCE SET content_warning = ${ContentWarning.MIXED.ordinal} WHERE is_nsfw = TRUE; + ALTER TABLE SOURCE DROP COLUMN is_nsfw; + + + """.trimIndent() + } + + fun postgresBackfill() = + """ + -- 1. Add the column as nullable to avoid table locks + ALTER TABLE EXTENSION ADD COLUMN extension_lib VARCHAR(16); + -- 2. Backfill existing rows using the first two parts of the version_name (split by dot) + UPDATE EXTENSION + SET extension_lib = CONCAT( + SPLIT_PART(version_name, '.', 1), + '.', + SPLIT_PART(version_name, '.', 2) + ); + -- 3. Enforce the NOT NULL constraint + ALTER TABLE EXTENSION ALTER COLUMN extension_lib SET NOT NULL; + """.trimIndent() + + fun h2Backfill() = + """ + -- 1. Add the column as nullable + ALTER TABLE EXTENSION ADD COLUMN extension_lib VARCHAR(16); + -- 2. Backfill rows by extracting text up to the second dot + UPDATE EXTENSION + SET extension_lib = CASE + -- If there's a second dot (e.g. 1.2.3), grab everything before it + WHEN LOCATE('.', version_name, LOCATE('.', version_name) + 1) > 0 + THEN SUBSTRING(version_name, 1, LOCATE('.', version_name, LOCATE('.', version_name) + 1) - 1) + -- If there's no second dot (e.g. 1.2), keep the original value + ELSE version_name + END; + -- 3. Enforce the NOT NULL constraint + ALTER TABLE EXTENSION ALTER COLUMN extension_lib SET NOT NULL; + """.trimIndent() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt new file mode 100644 index 000000000..dd4ba1af2 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt @@ -0,0 +1,32 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.AddTableMigration +import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.core.dao.id.IntIdTable + +@Suppress("ClassName", "unused") +class M0058_AddExtensionStore : AddTableMigration() { + private class ExtensionStoreTable : IntIdTable() { + val indexUrl = varchar("index_url", 2048).uniqueIndex() + val name = varchar("name", 256) + val badgeLabel = varchar("badge_label", 32) + val signingKey = varchar("signing_key", 512) + val contactWebsite = varchar("contact_website", 2048) + val contactDiscord = varchar("contact_discord", 2048).nullable() + val isLegacy = bool("is_legacy").default(false) + val extensionListUrl = varchar("extension_list_url", 2048).nullable() + } + + override val tables: Array + get() = + arrayOf( + ExtensionStoreTable(), + ) +} diff --git a/server/src/test/kotlin/masstest/TestExtensionCompatibility.kt b/server/src/test/kotlin/masstest/TestExtensionCompatibility.kt index cc569bb11..185fc7c68 100644 --- a/server/src/test/kotlin/masstest/TestExtensionCompatibility.kt +++ b/server/src/test/kotlin/masstest/TestExtensionCompatibility.kt @@ -48,7 +48,6 @@ class TestExtensionCompatibility { private val failedToFetch = mutableListOf>() private val mangaFailedToFetch = mutableListOf>() private val chaptersToFetch = mutableListOf>() - private val chaptersFailedToFetch = mutableListOf>() private val chaptersPageListFailedToFetch = mutableListOf, Exception>>() @BeforeAll @@ -133,10 +132,10 @@ class TestExtensionCompatibility { semaphore.withPermit { logger.info { "${mangaCount.getAndIncrement()} - Now fetching manga from $source" } try { - repeat { source.getMangaDetails(manga) } + repeat { source.getMangaUpdate(manga, emptyList(), true, true) } } catch (e: Exception) { logger.warn { - "Failed to fetch manga info from $source for ${manga.title} (${source.mangaDetailsRequest( + "Failed to fetch manga info and chapters from $source for ${manga.title} (${source.mangaDetailsRequest( manga, ).url}): ${e.message}" } @@ -154,50 +153,6 @@ class TestExtensionCompatibility { ) logger.info { "Now fetching manga chapters from ${mangaToFetch.size} sources" } - val chapterCount = AtomicInteger(1) - mangaToFetch - .filter { it.second.initialized } - .map { (source, manga) -> - async { - semaphore.withPermit { - logger.info { "${chapterCount.getAndIncrement()} - Now fetching manga chapters from $source" } - try { - chaptersToFetch += - Triple( - source, - manga, - repeat { - source.getChapterList(manga) - }.firstOrNull() - ?: throw Exception("Source returned no chapters"), - ) - } catch (e: Exception) { - logger.warn { - "Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest( - manga, - ).url}): ${e.message}" - } - chaptersFailedToFetch += Triple(source, manga, e) - } catch (e: NoClassDefFoundError) { - logger.warn { - "Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest( - manga, - ).url}): ${e.message}" - } - chaptersFailedToFetch += Triple(source, manga, e) - } - } - } - }.awaitAll() - - File("$BASE_PATH/ChaptersFailedToFetch.txt").writeText( - chaptersFailedToFetch.joinToString("\n") { (source, manga, exception) -> - "${source.name} (${source.lang}, ${source.id}):" + - " ${manga.title} (${source.mangaDetailsRequest(manga).url}):" + - " ${exception.message}" - }, - ) - val pageListCount = AtomicInteger(1) chaptersToFetch .map { (source, manga, chapter) -> diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/SearchTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/SearchTest.kt index 643310c5a..cbf6a2942 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/SearchTest.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/SearchTest.kt @@ -18,7 +18,6 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance -import rx.Observable import suwayomi.tachidesk.manga.impl.Search.FilterChange import suwayomi.tachidesk.manga.impl.Search.FilterObject import suwayomi.tachidesk.manga.impl.Search.SerializableGroup @@ -41,12 +40,11 @@ class SearchTest : ApplicationTest() { ) : StubSource(id) { var mangas: List = emptyList() - @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getSearchManga")) - override fun fetchSearchManga( + override suspend fun getSearchManga( page: Int, query: String, filters: FilterList, - ): Observable = Observable.just(MangasPage(mangas, false)) + ): MangasPage = MangasPage(mangas, false) } private val sourceId = 1L diff --git a/server/src/test/kotlin/suwayomi/tachidesk/test/TestUtils.kt b/server/src/test/kotlin/suwayomi/tachidesk/test/TestUtils.kt index de9a0c064..b04724c8b 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/test/TestUtils.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/test/TestUtils.kt @@ -66,6 +66,7 @@ fun createChapters( this[ChapterTable.sourceOrder] = it this[ChapterTable.isRead] = read this[ChapterTable.manga] = mangaId + this[ChapterTable.memo] = "{}" } } }