diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index ce622f1e7..7b7fa4c25 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -276,30 +276,38 @@ class ServerConfig( description = "Ignore re-uploaded chapters from auto-download", ) - val extensionRepos: MutableStateFlow> by ListSetting( + @Deprecated("Will get removed", replaceWith = ReplaceWith("extensionStores")) + val extensionRepos: MutableStateFlow> by MigratedConfigValue( protoNumber = 22, group = SettingGroup.EXTENSION, privacySafe = false, defaultValue = emptyList(), - itemValidator = { url -> - if (url.matches(repoMatchRegex)) { - null - } else { - "Invalid repository URL format" - } - }, - itemToValidValue = { url -> - if (url.matches(repoMatchRegex)) { - url - } else { - null - } - }, + deprecated = + SettingsRegistry.SettingDeprecated( + replaceWith = "extensionStores", + message = "Replaced with extensionStores", + migrateConfigValue = { + (it.unwrapped() as? List) + ?.map { + if (it.contains("github")) { + it.replace(repoMatchRegex) { + "https://raw.githubusercontent.com/${it.groupValues[2]}/${it.groupValues[3]}/" + + (it.groupValues.getOrNull(4)?.ifBlank { null } ?: "repo") + + "/" + + (it.groupValues.getOrNull(5)?.ifBlank { null } ?: "index.min.json") + } + } else { + it + } + } + }, + ), + readMigrated = { extensionStores.value }, + setMigrated = { extensionStores.value = it }, typeInfo = SettingsRegistry.PartialTypeInfo( specificType = "List", ), - description = "example: [\"https://github.com/MY_ACCOUNT/MY_REPO/tree/repo\"]", ) val maxSourcesInParallel: MutableStateFlow by IntSetting( @@ -1104,7 +1112,32 @@ class ServerConfig( privacySafe = true, defaultValue = false, description = "Skips the metadata feed and provides download/stream links directly in the chapter list. Improves compatibility with KOReader auto-downloader. KoSync strategies are applied, but PROMPT conflicts are ignored (treating local progress as priority)." + ) + val extensionStores: MutableStateFlow> by ListSetting( + protoNumber = 97, + group = SettingGroup.EXTENSION, + privacySafe = false, + defaultValue = emptyList(), + itemValidator = { url -> + if (url.isNotEmpty()) { + null + } else { + "Invalid store URL format" + } + }, + itemToValidValue = { url -> + if (url.isNotEmpty()) { + url + } else { + null + } + }, + typeInfo = + SettingsRegistry.PartialTypeInfo( + specificType = "List", + ), + description = "List of extension store index URLs", ) /** ****************************************************************** **/ diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt index bf102bd75..43258ab81 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt @@ -175,11 +175,12 @@ class LocalSource( chapters: List, fetchDetails: Boolean, fetchChapters: Boolean, - ): SMangaUpdate = supervisorScope { - val asyncManga = if (fetchDetails) async { getMangaDetails(manga) } else null - val asyncChapters = if (fetchChapters) async { getChapterList(manga) } else null - SMangaUpdate(asyncManga?.await() ?: manga, asyncChapters?.await() ?: chapters) - } + ): SMangaUpdate = + supervisorScope { + val asyncManga = if (fetchDetails) async { getMangaDetails(manga) } else null + val asyncChapters = if (fetchChapters) async { getChapterList(manga) } else null + SMangaUpdate(asyncManga?.await() ?: manga, asyncChapters?.await() ?: chapters) + } // Manga details related private suspend fun getMangaDetails(manga: SManga): SManga = @@ -481,7 +482,8 @@ class LocalSource( it[versionName] = "1.2" it[versionCode] = 0 it[lang] = LANG - it[isNsfw] = false + it[extensionLib] = "1.2" + it[contentRating] = 0 it[isInstalled] = true } @@ -490,7 +492,6 @@ class LocalSource( it[name] = NAME it[lang] = LANG it[extension] = extensionId - it[isNsfw] = false } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt new file mode 100644 index 000000000..eaa485685 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ExtensionStoreDataLoader.kt @@ -0,0 +1,57 @@ +package suwayomi.tachidesk.graphql.dataLoaders + +import com.expediagroup.graphql.dataloader.KotlinDataLoader +import graphql.GraphQLContext +import org.dataloader.DataLoader +import org.dataloader.DataLoaderFactory +import org.jetbrains.exposed.v1.core.Slf4jSqlDebugLogger +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.types.ExtensionNodeList +import suwayomi.tachidesk.graphql.types.ExtensionNodeList.Companion.toNodeList +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.graphql.types.ExtensionType +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.manga.model.table.ExtensionTable +import suwayomi.tachidesk.server.JavalinSetup.future + +class ExtensionStoreForExtension : KotlinDataLoader { + override val dataLoaderName = "ExtensionStoreForExtension" + + override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = + DataLoaderFactory.newDataLoader { ids -> + future { + transaction { + addLogger(Slf4jSqlDebugLogger) + val extensionStoreByIndexUrl = + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl inList ids } + .map { ExtensionStoreType(it) } + .associateBy { it.indexUrl } + ids.map { (extensionStoreByIndexUrl[it]) } + } + } + } +} + +class ExtensionForExtensionStore : KotlinDataLoader { + override val dataLoaderName = "ExtensionForExtensionStore" + + override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = + DataLoaderFactory.newDataLoader { ids -> + future { + transaction { + addLogger(Slf4jSqlDebugLogger) + val extensionByIndexUrl = + ExtensionTable + .selectAll() + .where { ExtensionTable.storeIndexUrl inList ids } + .map { ExtensionType(it) } + .groupBy { it.storeIndexUrl } + ids.map { (extensionByIndexUrl[it] ?: emptyList()).toNodeList() } + } + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt new file mode 100644 index 000000000..c3a39f007 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionStoreMutation.kt @@ -0,0 +1,104 @@ +package suwayomi.tachidesk.graphql.mutations + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.manga.impl.extension.ExtensionStoreService +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.server.JavalinSetup.future +import suwayomi.tachidesk.server.serverConfig +import java.util.concurrent.CompletableFuture + +class ExtensionStoreMutation { + data class AddExtensionStoreInput( + val clientMutationId: String? = null, + val indexUrl: String, + ) + + data class AddExtensionStorePayload( + val clientMutationId: String?, + val extensionStore: ExtensionStoreType, + ) + + @RequireAuth + fun addExtensionStore(input: AddExtensionStoreInput): CompletableFuture { + val (clientMutationId, indexUrl) = input + return future { + val store = ExtensionStoreService.fetch(indexUrl) + + ExtensionStoreService.upsert(store) + serverConfig.extensionStores.value = (serverConfig.extensionStores.value + indexUrl).distinct() + val row = + transaction { + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq store.indexUrl } + .first() + } + + AddExtensionStorePayload( + clientMutationId = clientMutationId, + extensionStore = ExtensionStoreType(row), + ) + } + } + + data class RemoveExtensionStoreInput( + val clientMutationId: String? = null, + val indexUrl: String, + ) + + data class RemoveExtensionStorePayload( + val clientMutationId: String?, + val extensionStore: ExtensionStoreType?, + ) + + @RequireAuth + fun removeExtensionStore(input: RemoveExtensionStoreInput): CompletableFuture { + val (clientMutationId, indexUrl) = input + return future { + val store = + transaction { + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq indexUrl } + .firstOrNull() + ?.let { ExtensionStoreType(it) } + } + + store?.let { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl eq indexUrl } + } + } + + serverConfig.extensionStores.value = serverConfig.extensionStores.value.filterNot { it == indexUrl } + + RemoveExtensionStorePayload( + clientMutationId = clientMutationId, + extensionStore = + store?.let { + ExtensionStoreType( + name = it.name, + badgeLabel = it.badgeLabel, + signingKey = it.signingKey, + contactWebsite = it.contactWebsite, + contactDiscord = it.contactDiscord, + indexUrl = it.indexUrl, + isLegacy = it.isLegacy, + ) + }, + ) + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt index d3bf9b196..39011fdf3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/MangaMutation.kt @@ -190,12 +190,13 @@ class MangaMutation { var mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq id }.first() } val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) - val sMangaUpdate = Manga.fetchMangaAndChapters( - mangaEntry = mangaEntry, - source = source, - fetchDetails = fetchManga, - fetchChapters = fetchChapters - ) + val sMangaUpdate = + Manga.fetchMangaAndChapters( + mangaEntry = mangaEntry, + source = source, + fetchDetails = fetchManga, + fetchChapters = fetchChapters, + ) Manga.updateMangaDatabase(mangaEntry, source, sMangaUpdate.manga) mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq id }.first() } @@ -209,9 +210,8 @@ class MangaMutation { .selectAll() .where { ChapterTable.manga eq id } .orderBy(ChapterTable.sourceOrder) - .map { ChapterType(it) } + .map { ChapterType(it) }, ) - } FetchMangaAndChaptersPayload( clientMutationId = clientMutationId, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt index c2b1a8da6..615609892 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt @@ -24,6 +24,7 @@ import suwayomi.tachidesk.graphql.queries.filter.BooleanFilter import suwayomi.tachidesk.graphql.queries.filter.Filter import suwayomi.tachidesk.graphql.queries.filter.HasGetOp import suwayomi.tachidesk.graphql.queries.filter.IntFilter +import suwayomi.tachidesk.graphql.queries.filter.LongFilter import suwayomi.tachidesk.graphql.queries.filter.OpAnd import suwayomi.tachidesk.graphql.queries.filter.StringFilter import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompare @@ -38,6 +39,7 @@ import suwayomi.tachidesk.graphql.server.primitives.applyBeforeAfter import suwayomi.tachidesk.graphql.server.primitives.greaterNotUnique import suwayomi.tachidesk.graphql.server.primitives.lessNotUnique import suwayomi.tachidesk.graphql.server.primitives.maybeSwap +import suwayomi.tachidesk.graphql.types.ContentRating import suwayomi.tachidesk.graphql.types.ExtensionNodeList import suwayomi.tachidesk.graphql.types.ExtensionType import suwayomi.tachidesk.manga.model.table.ExtensionTable @@ -55,21 +57,23 @@ class ExtensionQuery { ) : OrderBy { PKG_NAME(ExtensionTable.pkgName), NAME(ExtensionTable.name), - APK_NAME(ExtensionTable.apkName), + + @GraphQLDeprecated("") + APK_NAME(ExtensionTable.pkgName), ; override fun greater(cursor: Cursor): Op = when (this) { PKG_NAME -> ExtensionTable.pkgName greater cursor.value NAME -> greaterNotUnique(ExtensionTable.name, ExtensionTable.pkgName, cursor, String::toString) - APK_NAME -> greaterNotUnique(ExtensionTable.apkName, ExtensionTable.pkgName, cursor, String::toString) + APK_NAME -> ExtensionTable.pkgName greater cursor.value } override fun less(cursor: Cursor): Op = when (this) { PKG_NAME -> ExtensionTable.pkgName less cursor.value NAME -> lessNotUnique(ExtensionTable.name, ExtensionTable.pkgName, cursor, String::toString) - APK_NAME -> lessNotUnique(ExtensionTable.apkName, ExtensionTable.pkgName, cursor, String::toString) + APK_NAME -> ExtensionTable.pkgName less cursor.value } override fun asCursor(type: ExtensionType): Cursor { @@ -89,29 +93,41 @@ class ExtensionQuery { ) : Order data class ExtensionCondition( + val storeIndexUrl: String? = null, + @GraphQLDeprecated("", ReplaceWith("storeIndexUrl")) val repo: String? = null, val apkName: String? = null, val iconUrl: String? = null, val name: String? = null, val pkgName: String? = null, + val apkUrl: String? = null, + val extensionLib: String? = null, val versionName: String? = null, val versionCode: Int? = null, + val versionCodeLong: Long? = null, val lang: String? = null, + @GraphQLDeprecated("", ReplaceWith("contentRating")) val isNsfw: Boolean? = null, + val contentRating: ContentRating? = null, val isInstalled: Boolean? = null, val hasUpdate: Boolean? = null, val isObsolete: Boolean? = null, ) : HasGetOp { override fun getOp(): Op? { val opAnd = OpAnd() - opAnd.eq(repo, ExtensionTable.repo) + opAnd.eq(storeIndexUrl, ExtensionTable.storeIndexUrl) + opAnd.eq(repo, ExtensionTable.storeIndexUrl) opAnd.eq(apkName, ExtensionTable.apkName) opAnd.eq(iconUrl, ExtensionTable.iconUrl) + opAnd.eq(apkUrl, ExtensionTable.apkUrl) opAnd.eq(name, ExtensionTable.name) + opAnd.eq(extensionLib, ExtensionTable.extensionLib) opAnd.eq(versionName, ExtensionTable.versionName) - opAnd.eq(versionCode, ExtensionTable.versionCode) + opAnd.eq(versionCode?.toLong(), ExtensionTable.versionCode) + opAnd.eq(versionCodeLong, ExtensionTable.versionCode) opAnd.eq(lang, ExtensionTable.lang) - opAnd.eq(isNsfw, ExtensionTable.isNsfw) + opAnd.eq(isNsfw?.let { if (it) 3 else 0 }, ExtensionTable.contentRating) + opAnd.eq(contentRating?.ordinal, ExtensionTable.contentRating) opAnd.eq(isInstalled, ExtensionTable.isInstalled) opAnd.eq(hasUpdate, ExtensionTable.hasUpdate) opAnd.eq(isObsolete, ExtensionTable.isObsolete) @@ -121,15 +137,23 @@ class ExtensionQuery { } data class ExtensionFilter( + val storeIndexUrl: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("storeIndexUrl")) val repo: StringFilter? = null, val apkName: StringFilter? = null, val iconUrl: StringFilter? = null, val name: StringFilter? = null, val pkgName: StringFilter? = null, + val apkUrl: StringFilter? = null, val versionName: StringFilter? = null, + val extensionLib: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("versionCodeLong")) val versionCode: IntFilter? = null, + val versionCodeLong: LongFilter? = null, val lang: StringFilter? = null, + @GraphQLDeprecated("", ReplaceWith("storeIndexUrl")) val isNsfw: BooleanFilter? = null, + // val contentRating: EnumFilter? = null, val isInstalled: BooleanFilter? = null, val hasUpdate: BooleanFilter? = null, val isObsolete: BooleanFilter? = null, @@ -139,15 +163,18 @@ class ExtensionQuery { ) : Filter { override fun getOpList(): List> = listOfNotNull( - andFilterWithCompareString(ExtensionTable.repo, repo), + andFilterWithCompareString(ExtensionTable.storeIndexUrl, storeIndexUrl), + andFilterWithCompareString(ExtensionTable.storeIndexUrl, repo), andFilterWithCompareString(ExtensionTable.apkName, apkName), andFilterWithCompareString(ExtensionTable.iconUrl, iconUrl), andFilterWithCompareString(ExtensionTable.name, name), andFilterWithCompareString(ExtensionTable.pkgName, pkgName), + andFilterWithCompareString(ExtensionTable.apkUrl, apkUrl), + andFilterWithCompareString(ExtensionTable.extensionLib, extensionLib), andFilterWithCompareString(ExtensionTable.versionName, versionName), - andFilterWithCompare(ExtensionTable.versionCode, versionCode), + andFilterWithCompare(ExtensionTable.versionCode, versionCodeLong), andFilterWithCompareString(ExtensionTable.lang, lang), - andFilterWithCompare(ExtensionTable.isNsfw, isNsfw), + // andFilterWithCompareEnum(ExtensionTable.contentRating, contentRating), andFilterWithCompare(ExtensionTable.isInstalled, isInstalled), andFilterWithCompare(ExtensionTable.hasUpdate, hasUpdate), andFilterWithCompare(ExtensionTable.isObsolete, isObsolete), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt new file mode 100644 index 000000000..effb26142 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionStoreQuery.kt @@ -0,0 +1,39 @@ +package suwayomi.tachidesk.graphql.queries + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.ExtensionStoreType +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import java.util.concurrent.CompletableFuture + +class ExtensionStoreQuery { + @RequireAuth + fun extensionStore(indexUrl: String): CompletableFuture = + CompletableFuture.supplyAsync { + transaction { + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq indexUrl } + .firstOrNull() + ?.let { ExtensionStoreType(it) } + } + } + + @RequireAuth + fun extensionStores(): List = + transaction { + ExtensionStoreTable + .selectAll() + .toList() + .map { ExtensionStoreType(it) } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt index 216ee41b4..810c40529 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SourceQuery.kt @@ -13,7 +13,9 @@ import graphql.schema.DataFetchingEnvironment import org.jetbrains.exposed.v1.core.Column import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.greater +import org.jetbrains.exposed.v1.core.greaterEq import org.jetbrains.exposed.v1.core.less import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -24,7 +26,6 @@ import suwayomi.tachidesk.graphql.queries.filter.HasGetOp import suwayomi.tachidesk.graphql.queries.filter.LongFilter import suwayomi.tachidesk.graphql.queries.filter.OpAnd import suwayomi.tachidesk.graphql.queries.filter.StringFilter -import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompare import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareEntity import suwayomi.tachidesk.graphql.queries.filter.andFilterWithCompareString import suwayomi.tachidesk.graphql.queries.filter.applyOps @@ -37,6 +38,7 @@ import suwayomi.tachidesk.graphql.server.primitives.applyBeforeAfter import suwayomi.tachidesk.graphql.server.primitives.greaterNotUnique import suwayomi.tachidesk.graphql.server.primitives.lessNotUnique import suwayomi.tachidesk.graphql.server.primitives.maybeSwap +import suwayomi.tachidesk.graphql.types.ContentRating import suwayomi.tachidesk.graphql.types.SourceNodeList import suwayomi.tachidesk.graphql.types.SourceType import suwayomi.tachidesk.manga.model.table.SourceTable @@ -91,14 +93,17 @@ class SourceQuery { val id: Long? = null, val name: String? = null, val lang: String? = null, + @GraphQLDeprecated("replace with contentRating == 3", ReplaceWith("contentRating")) val isNsfw: Boolean? = null, + val contentRating: ContentRating? = null, ) : HasGetOp { override fun getOp(): Op? { val opAnd = OpAnd() opAnd.eq(id, SourceTable.id) opAnd.eq(name, SourceTable.name) opAnd.eq(lang, SourceTable.lang) - opAnd.eq(isNsfw, SourceTable.isNsfw) + opAnd.andWhere(isNsfw) { if (it) SourceTable.contentRating greaterEq 3 else SourceTable.contentRating less 3 } + opAnd.andWhere(contentRating) { SourceTable.contentRating eq it.getValue() } return opAnd.op } @@ -108,7 +113,9 @@ class SourceQuery { val id: LongFilter? = null, val name: StringFilter? = null, val lang: StringFilter? = null, + @GraphQLDeprecated("replace with contentRating == 3", ReplaceWith("contentRating")) val isNsfw: BooleanFilter? = null, + // val contentRating: EnumFilter? = null, override val and: List? = null, override val or: List? = null, override val not: SourceFilter? = null, @@ -118,7 +125,7 @@ class SourceQuery { andFilterWithCompareEntity(SourceTable.id, id), andFilterWithCompareString(SourceTable.name, name), andFilterWithCompareString(SourceTable.lang, lang), - andFilterWithCompare(SourceTable.isNsfw, isNsfw), + // andFilterWithCompareEnum(SourceTable.contentRating, contentRating) ) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt index 74998b6c1..401a48797 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/filter/Filter.kt @@ -329,6 +329,24 @@ data class DoubleFilter( ) } +data class EnumFilter>( + override val isNull: Boolean? = null, + override val equalTo: T? = null, + override val notEqualTo: T? = null, + override val notEqualToAll: List? = null, + override val notEqualToAny: List? = null, + override val distinctFrom: T? = null, + override val distinctFromAll: List? = null, + override val distinctFromAny: List? = null, + override val notDistinctFrom: T? = null, + override val `in`: List? = null, + override val notIn: List? = null, + override val lessThan: T? = null, + override val lessThanOrEqualTo: T? = null, + override val greaterThan: T? = null, + override val greaterThanOrEqualTo: T? = null, +) : ComparableScalarFilter + data class StringFilter( override val isNull: Boolean? = null, override val equalTo: String? = null, @@ -618,6 +636,35 @@ fun , S : T?> andFilterWithCompare( return opAnd.op } +@Suppress("UNCHECKED_CAST") +fun > andFilterWithCompareEnum( + column: Column, + filter: ComparableScalarFilter?, +): Op? { + filter ?: return null + val opAnd = OpAnd() + + opAnd.andWhere(filter.lessThan) { column less it.ordinal } + opAnd.andWhere(filter.lessThanOrEqualTo) { column lessEq it.ordinal } + opAnd.andWhere(filter.greaterThan) { column greater it.ordinal } + opAnd.andWhere(filter.greaterThanOrEqualTo) { column greaterEq it.ordinal } + + opAnd.andWhere(filter.isNull) { if (it) column.isNull() else column.isNotNull() } + + opAnd.andWhere(filter.equalTo) { column eq it.ordinal } + opAnd.andNotWhere(filter.notEqualTo, filter.notEqualToAll, filter.notEqualToAny) { column neq it.ordinal } + opAnd.andWhere(filter.distinctFrom, filter.distinctFromAll, filter.distinctFromAny) { DistinctFromOp.distinctFrom(column, it.ordinal) } + opAnd.andWhere(filter.notDistinctFrom) { DistinctFromOp.notDistinctFrom(column, it.ordinal) } + if (!filter.`in`.isNullOrEmpty()) { + opAnd.andWhere(filter.`in`) { column inList it.map { it.ordinal } } + } + if (!filter.notIn.isNullOrEmpty()) { + opAnd.andWhere(filter.notIn) { column notInList it.map { it.ordinal } } + } + + return opAnd.op +} + fun > andFilterWithCompareEntity( column: Column>, filter: ComparableScalarFilter?, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt index 4ff568eea..60dbfc151 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt @@ -20,7 +20,9 @@ import suwayomi.tachidesk.graphql.dataLoaders.DisplayScoreForTrackRecordDataLoad import suwayomi.tachidesk.graphql.dataLoaders.DisplayScoreForTrackSearchDataLoader import suwayomi.tachidesk.graphql.dataLoaders.DownloadedChapterCountForMangaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.ExtensionDataLoader +import suwayomi.tachidesk.graphql.dataLoaders.ExtensionForExtensionStore import suwayomi.tachidesk.graphql.dataLoaders.ExtensionForSourceDataLoader +import suwayomi.tachidesk.graphql.dataLoaders.ExtensionStoreForExtension import suwayomi.tachidesk.graphql.dataLoaders.FirstUnreadChapterForMangaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.GlobalMetaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.HasDuplicateChaptersForMangaDataLoader @@ -78,6 +80,8 @@ class TachideskDataLoaderRegistryFactory { SourceMetaDataLoader(), ExtensionDataLoader(), ExtensionForSourceDataLoader(), + ExtensionForExtensionStore(), + ExtensionStoreForExtension(), TrackerDataLoader(), TrackerStatusesDataLoader(), TrackerScoresDataLoader(), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt index d6527ae3a..1ee37b97b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt @@ -20,6 +20,7 @@ import suwayomi.tachidesk.graphql.mutations.CategoryMutation import suwayomi.tachidesk.graphql.mutations.ChapterMutation import suwayomi.tachidesk.graphql.mutations.DownloadMutation import suwayomi.tachidesk.graphql.mutations.ExtensionMutation +import suwayomi.tachidesk.graphql.mutations.ExtensionStoreMutation import suwayomi.tachidesk.graphql.mutations.ImageMutation import suwayomi.tachidesk.graphql.mutations.InfoMutation import suwayomi.tachidesk.graphql.mutations.KoreaderSyncMutation @@ -36,6 +37,7 @@ import suwayomi.tachidesk.graphql.queries.CategoryQuery import suwayomi.tachidesk.graphql.queries.ChapterQuery import suwayomi.tachidesk.graphql.queries.DownloadQuery import suwayomi.tachidesk.graphql.queries.ExtensionQuery +import suwayomi.tachidesk.graphql.queries.ExtensionStoreQuery import suwayomi.tachidesk.graphql.queries.InfoQuery import suwayomi.tachidesk.graphql.queries.KoreaderSyncQuery import suwayomi.tachidesk.graphql.queries.MangaQuery @@ -95,6 +97,7 @@ val schema = TopLevelObject(ChapterQuery()), TopLevelObject(DownloadQuery()), TopLevelObject(ExtensionQuery()), + TopLevelObject(ExtensionStoreQuery()), TopLevelObject(InfoQuery()), TopLevelObject(KoreaderSyncQuery()), TopLevelObject(MangaQuery()), @@ -112,6 +115,7 @@ val schema = TopLevelObject(ChapterMutation()), TopLevelObject(DownloadMutation()), TopLevelObject(ExtensionMutation()), + TopLevelObject(ExtensionStoreMutation()), TopLevelObject(ImageMutation()), TopLevelObject(InfoMutation()), TopLevelObject(KoreaderSyncMutation()), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt new file mode 100644 index 000000000..89ec32600 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionStoreType.kt @@ -0,0 +1,38 @@ +package suwayomi.tachidesk.graphql.types + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import com.expediagroup.graphql.server.extensions.getValueFromDataLoader +import graphql.schema.DataFetchingEnvironment +import org.jetbrains.exposed.v1.core.ResultRow +import suwayomi.tachidesk.graphql.server.primitives.Node +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import java.util.concurrent.CompletableFuture + +class ExtensionStoreType( + val name: String, + val badgeLabel: String, + val signingKey: String, + val contactWebsite: String, + val contactDiscord: String?, + val indexUrl: String, + val isLegacy: Boolean, +) : Node { + constructor(row: ResultRow) : this( + name = row[ExtensionStoreTable.name], + badgeLabel = row[ExtensionStoreTable.badgeLabel], + signingKey = row[ExtensionStoreTable.signingKey], + contactWebsite = row[ExtensionStoreTable.contactWebsite], + contactDiscord = row[ExtensionStoreTable.contactDiscord], + indexUrl = row[ExtensionStoreTable.indexUrl], + isLegacy = row[ExtensionStoreTable.isLegacy], + ) + + fun extension(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = + dataFetchingEnvironment.getValueFromDataLoader("ExtensionForExtensionStore", indexUrl) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt index 97d6a59ac..68570f7e0 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt @@ -7,6 +7,8 @@ package suwayomi.tachidesk.graphql.types +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated +import com.expediagroup.graphql.generator.annotations.GraphQLDescription import com.expediagroup.graphql.server.extensions.getValueFromDataLoader import graphql.schema.DataFetchingEnvironment import org.jetbrains.exposed.v1.core.ResultRow @@ -20,29 +22,46 @@ import suwayomi.tachidesk.manga.model.table.ExtensionTable import java.util.concurrent.CompletableFuture class ExtensionType( + val storeIndexUrl: String?, + @GraphQLDeprecated("Removed in extension api v1.6", ReplaceWith("storeIndexUrl")) val repo: String?, - val apkName: String, + @GraphQLDescription("This will be nullable in the future") + val apkName: String?, val iconUrl: String, val name: String, val pkgName: String, + val apkUrl: String?, + val extensionLib: String?, val versionName: String, + @GraphQLDeprecated( + "Type was changed to Long, will be switched back to this variable name in the future.", + ReplaceWith("versionCodeLong"), + ) val versionCode: Int, + val versionCodeLong: Long, val lang: String, + @GraphQLDeprecated("Removed in extension api v1.6", ReplaceWith("contentRating")) val isNsfw: Boolean, + val contentRating: ContentRating, val isInstalled: Boolean, val hasUpdate: Boolean, val isObsolete: Boolean, ) : Node { constructor(row: ResultRow) : this( - repo = row[ExtensionTable.repo], - apkName = row[ExtensionTable.apkName], - iconUrl = Extension.getExtensionIconUrl(row[ExtensionTable.apkName]), + storeIndexUrl = row[ExtensionTable.storeIndexUrl], + repo = row[ExtensionTable.storeIndexUrl], + apkName = row[ExtensionTable.apkName].orEmpty(), + iconUrl = Extension.proxyExtensionIconUrl(row[ExtensionTable.pkgName]), name = row[ExtensionTable.name], pkgName = row[ExtensionTable.pkgName], + apkUrl = row[ExtensionTable.apkUrl], + extensionLib = row[ExtensionTable.extensionLib], versionName = row[ExtensionTable.versionName], - versionCode = row[ExtensionTable.versionCode], + versionCode = row[ExtensionTable.versionCode].toInt(), + versionCodeLong = row[ExtensionTable.versionCode], lang = row[ExtensionTable.lang], - isNsfw = row[ExtensionTable.isNsfw], + isNsfw = row[ExtensionTable.contentRating] == 3, + contentRating = ContentRating.valueOf(row[ExtensionTable.contentRating]), isInstalled = row[ExtensionTable.isInstalled], hasUpdate = row[ExtensionTable.hasUpdate], isObsolete = row[ExtensionTable.isObsolete], @@ -50,6 +69,9 @@ class ExtensionType( fun source(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = dataFetchingEnvironment.getValueFromDataLoader("SourcesForExtensionDataLoader", pkgName) + + fun extensionStore(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = + dataFetchingEnvironment.getValueFromDataLoader("ExtensionStoreForExtension", storeIndexUrl.orEmpty()) } data class ExtensionNodeList( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt index f6c4744a2..a10c430e4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SourceType.kt @@ -7,6 +7,8 @@ package suwayomi.tachidesk.graphql.types +import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated +import com.expediagroup.graphql.generator.annotations.GraphQLIgnore import com.expediagroup.graphql.server.extensions.getValueFromDataLoader import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.ConfigurableSource @@ -25,7 +27,6 @@ import suwayomi.tachidesk.manga.impl.Source.getSourcePreferencesRaw import suwayomi.tachidesk.manga.impl.extension.Extension import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub -import suwayomi.tachidesk.manga.model.dataclass.SourceDataClass import suwayomi.tachidesk.manga.model.table.ExtensionTable import suwayomi.tachidesk.manga.model.table.SourceTable import java.util.concurrent.CompletableFuture @@ -41,35 +42,31 @@ class SourceType( val id: Long, val name: String, val lang: String, + val message: String?, + val contentRating: ContentRating, val iconUrl: String, val supportsLatest: Boolean, val isConfigurable: Boolean, + @GraphQLDeprecated("", ReplaceWith("contentRating")) val isNsfw: Boolean, val displayName: String, + val homeUrl: String?, + @GraphQLDeprecated("", ReplaceWith("homeUrl")) val baseUrl: String?, ) : Node { - constructor(source: SourceDataClass) : this( - id = source.id.toLong(), - name = source.name, - lang = source.lang, - iconUrl = source.iconUrl, - supportsLatest = source.supportsLatest, - isConfigurable = source.isConfigurable, - isNsfw = source.isNsfw, - displayName = source.displayName, - baseUrl = source.baseUrl, - ) - constructor(row: ResultRow, sourceExtension: ResultRow, catalogueSource: CatalogueSource) : this( id = row[SourceTable.id].value, name = row[SourceTable.name], lang = row[SourceTable.lang], - iconUrl = Extension.getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]), + message = row[SourceTable.message], + contentRating = ContentRating.valueOf(row[SourceTable.contentRating]), + iconUrl = Extension.proxyExtensionIconUrl(sourceExtension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = row[SourceTable.isNsfw], + isNsfw = row[SourceTable.contentRating] >= 3, displayName = catalogueSource.toString(), - baseUrl = catalogueSource.runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), + homeUrl = runCatching { (catalogueSource as? HttpSource)?.getHomeUrl() }.getOrNull(), + baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) fun manga(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture = @@ -510,3 +507,33 @@ fun preferenceOf(preference: SourcePreference): Preference = throw RuntimeException("sealed class cannot have more subtypes!") } } + +enum class ContentRating { + SAFE, + SUGGESTIVE, + EROTICA, + PORNOGRAPHIC, + ; + + @GraphQLIgnore + fun getValue() = + when (this) { + SAFE -> 0 + SUGGESTIVE -> 1 + EROTICA -> 2 + PORNOGRAPHIC -> 3 + } + + @GraphQLIgnore + companion object { + @GraphQLIgnore + fun valueOf(contentRating: Int) = + when (contentRating) { + 0 -> SAFE + 1 -> SUGGESTIVE + 2 -> EROTICA + 3 -> PORNOGRAPHIC + else -> SAFE + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt index 89b8f3dfc..2da3c3740 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt @@ -34,7 +34,7 @@ object MangaAPI { get("update/{pkgName}", ExtensionController.update) get("uninstall/{pkgName}", ExtensionController.uninstall) - get("icon/{apkName}", ExtensionController.icon) + get("icon/{pkgName}", ExtensionController.icon) } path("source") { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt index 006b27adb..220864dab 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt @@ -165,17 +165,17 @@ object ExtensionController { /** icon for extension named `apkName` */ val icon = handler( - pathParam("apkName"), + pathParam("pkgName"), documentWith = { withOperation { summary("Extension icon") description("Icon for extension named `apkName`") } }, - behaviorOf = { ctx, apkName -> + behaviorOf = { ctx, pkgName -> ctx.getAttribute(Attribute.TachideskUser).requireUser() ctx.future { - future { Extension.getExtensionIcon(apkName) } + future { Extension.getExtensionIcon(pkgName) } .thenApply { ctx.header("content-type", it.second) val httpCacheSeconds = 365.days.inWholeSeconds diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index 5c075baba..d2758c324 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -18,6 +18,7 @@ import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and @@ -114,17 +115,20 @@ object Chapter { val mutex = Manga.mangaInfoMutex.get(mangaId) { Mutex() } val chapterList = mutex.withLock { - val mangaEntry = transaction { - MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() - } + val mangaEntry = + transaction { + MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() + } val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) - val chapters = Manga.fetchMangaAndChapters( - mangaEntry = mangaEntry, - source = source, - fetchDetails = false, - fetchChapters = true, - ).chapters + val chapters = + Manga + .fetchMangaAndChapters( + mangaEntry = mangaEntry, + source = source, + fetchDetails = false, + fetchChapters = true, + ).chapters updateChapterListDatabase(mangaEntry, chapters, source) } @@ -150,22 +154,28 @@ object Chapter { } // Recognize number for new chapters. - val sManga = SManga.create().apply { - url = mangaEntry[MangaTable.url] - title = mangaEntry[MangaTable.title] - thumbnail_url = mangaEntry[MangaTable.thumbnail_url] - artist = mangaEntry[MangaTable.artist] - author = mangaEntry[MangaTable.author] - description = mangaEntry[MangaTable.description] - genre = mangaEntry[MangaTable.genre] - status = mangaEntry[MangaTable.status] - update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) - memo = mangaEntry[MangaTable.memo] - initialized = mangaEntry[MangaTable.initialized] - } + val sManga = + SManga.create().apply { + url = mangaEntry[MangaTable.url] + title = mangaEntry[MangaTable.title] + thumbnail_url = mangaEntry[MangaTable.thumbnail_url] + artist = mangaEntry[MangaTable.artist] + author = mangaEntry[MangaTable.author] + description = mangaEntry[MangaTable.description] + genre = mangaEntry[MangaTable.genre] + status = mangaEntry[MangaTable.status] + update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) + memo = Json.decodeFromString(mangaEntry[MangaTable.memo]) + initialized = mangaEntry[MangaTable.initialized] + } uniqueChapters.forEach { chapter -> (source as? HttpSource)?.prepareNewChapter(chapter, sManga) - val chapterNumber = ChapterRecognition.parseChapterNumber(mangaEntry[MangaTable.title], chapter.name, chapter.chapter_number.toDouble()) + val chapterNumber = + ChapterRecognition.parseChapterNumber( + mangaEntry[MangaTable.title], + chapter.name, + chapter.chapter_number.toDouble(), + ) chapter.chapter_number = chapterNumber.toFloat() chapter.name = chapter.name.sanitize(mangaEntry[MangaTable.title]) chapter.scanlator = chapter.scanlator?.ifBlank { null }?.trim() @@ -270,7 +280,7 @@ object Chapter { this[ChapterTable.fetchedAt] = chapter.fetchedAt this[ChapterTable.manga] = chapter.mangaId this[ChapterTable.realUrl] = chapter.realUrl - this[ChapterTable.memo] = chapter.memo + this[ChapterTable.memo] = Json.encodeToString(chapter.memo) this[ChapterTable.isRead] = false this[ChapterTable.isBookmarked] = false this[ChapterTable.isDownloaded] = false @@ -321,7 +331,7 @@ object Chapter { this[ChapterTable.realUrl] = it.realUrl this[ChapterTable.lastModifiedAt] = it.lastModifiedAt this[ChapterTable.version] = it.version - this[ChapterTable.memo] = it.memo + this[ChapterTable.memo] = Json.encodeToString(it.memo) this[ChapterTable.isDownloaded] = currentChapter.downloaded this[ChapterTable.pageCount] = currentChapter.pageCount @@ -359,7 +369,7 @@ object Chapter { mangaEntry[MangaTable.id].value, currentLatestChapterNumber, numberOfCurrentChapters, - insertedChapters + insertedChapters, ) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index de1c7070e..2684c29f7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -24,6 +24,7 @@ import io.github.reactivecircus.cache4k.Cache import io.javalin.http.HttpStatus import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.json.Json import okhttp3.CacheControl import okhttp3.Response import org.jetbrains.exposed.v1.core.ResultRow @@ -105,24 +106,26 @@ object Manga { genre = mangaEntry[MangaTable.genre] status = mangaEntry[MangaTable.status] update_strategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]) - memo = mangaEntry[MangaTable.memo] + memo = Json.decodeFromString(mangaEntry[MangaTable.memo]) initialized = mangaEntry[MangaTable.initialized] } - val sChapters = transaction { - ChapterTable.selectAll() - .where { ChapterTable.manga eq mangaEntry[MangaTable.id] } - .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) - .map { - SChapter.create().apply { - url = it[ChapterTable.url] - name = it[ChapterTable.name] - chapter_number = it[ChapterTable.chapter_number] - scanlator = it[ChapterTable.scanlator] - date_upload = it[ChapterTable.date_upload] - memo = it[ChapterTable.memo] + val sChapters = + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq mangaEntry[MangaTable.id] } + .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) + .map { + SChapter.create().apply { + url = it[ChapterTable.url] + name = it[ChapterTable.name] + chapter_number = it[ChapterTable.chapter_number] + scanlator = it[ChapterTable.scanlator] + date_upload = it[ChapterTable.date_upload] + memo = Json.decodeFromString(it[ChapterTable.memo]) + } } - } - } + } return source.getMangaUpdate( sManga, @@ -137,12 +140,13 @@ object Manga { val mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } val source = getCatalogueSourceOrNull(mangaEntry[MangaTable.sourceReference]) ?: return null - val sManga = fetchMangaAndChapters( - mangaEntry, - source, - fetchDetails = true, - fetchChapters = false - ).manga + val sManga = + fetchMangaAndChapters( + mangaEntry, + source, + fetchDetails = true, + fetchChapters = false, + ).manga updateMangaDatabase(mangaEntry, source, sManga) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt index 9d405c506..b7d888ec3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt @@ -9,6 +9,7 @@ package suwayomi.tachidesk.manga.impl import eu.kanade.tachiyomi.source.local.LocalSource import eu.kanade.tachiyomi.source.model.MangasPage +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.dao.id.EntityID import org.jetbrains.exposed.v1.core.eq @@ -75,6 +76,7 @@ object MangaList { this[MangaTable.status] = it.status this[MangaTable.thumbnail_url] = it.thumbnail_url this[MangaTable.updateStrategy] = it.update_strategy.name + this[MangaTable.memo] = Json.encodeToString(it.memo) this[MangaTable.sourceReference] = sourceId }.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) } @@ -103,6 +105,7 @@ object MangaList { this[MangaTable.status] = sManga.status this[MangaTable.thumbnail_url] = sManga.thumbnail_url ?: manga[MangaTable.thumbnail_url] this[MangaTable.updateStrategy] = sManga.update_strategy.name + this[MangaTable.memo] = Json.encodeToString(sManga.memo) if (!sManga.thumbnail_url.isNullOrEmpty() && manga[MangaTable.thumbnail_url] != sManga.thumbnail_url) { this[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond Manga.clearThumbnail(manga[MangaTable.id].value) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt index fbf5e6d12..0631b6948 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt @@ -25,7 +25,7 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import suwayomi.tachidesk.manga.impl.Source.preferenceScreenMap -import suwayomi.tachidesk.manga.impl.extension.Extension.getExtensionIconUrl +import suwayomi.tachidesk.manga.impl.extension.Extension.proxyExtensionIconUrl import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.unregisterCatalogueSource @@ -49,10 +49,10 @@ object Source { id = it[SourceTable.id].value.toString(), name = it[SourceTable.name], lang = it[SourceTable.lang], - iconUrl = getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]), + iconUrl = proxyExtensionIconUrl(sourceExtension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = it[SourceTable.isNsfw], + isNsfw = it[SourceTable.contentRating] >= 3, displayName = catalogueSource.toString(), baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) @@ -70,13 +70,10 @@ object Source { id = sourceId.toString(), name = source[SourceTable.name], lang = source[SourceTable.lang], - iconUrl = - getExtensionIconUrl( - extension[ExtensionTable.apkName], - ), + iconUrl = proxyExtensionIconUrl(extension[ExtensionTable.pkgName]), supportsLatest = catalogueSource.supportsLatest, isConfigurable = catalogueSource is ConfigurableSource, - isNsfw = source[SourceTable.isNsfw], + isNsfw = source[SourceTable.contentRating] >= 3, displayName = catalogueSource.toString(), baseUrl = runCatching { (catalogueSource as? HttpSource)?.baseUrl }.getOrNull(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt index 8bbf54433..02343faba 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt @@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl.backup.proto.handlers import eu.kanade.tachiyomi.source.model.UpdateStrategy import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonObject import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and @@ -79,7 +78,7 @@ object BackupMangaHandler { lastModifiedAt = mangaRow[MangaTable.lastModifiedAt], version = mangaRow[MangaTable.version], initialized = mangaRow[MangaTable.initialized], - memo = Json.encodeToString(mangaRow[MangaTable.memo]).encodeToByteArray() + memo = Json.encodeToString(mangaRow[MangaTable.memo]).encodeToByteArray(), ) val mangaId = mangaRow[MangaTable.id].value @@ -242,7 +241,7 @@ object BackupMangaHandler { it[lastModifiedAt] = manga.lastModifiedAt it[version] = manga.version - it[memo] = Json.decodeFromString(manga.memo.decodeToString()) + it[memo] = manga.memo.decodeToString() }.value } else { val dbMangaId = dbManga[MangaTable.id].value @@ -265,7 +264,7 @@ object BackupMangaHandler { it[lastModifiedAt] = manga.lastModifiedAt it[version] = manga.version - it[memo] = Json.decodeFromString(manga.memo.decodeToString()) + it[memo] = manga.memo.decodeToString() } dbMangaId @@ -357,7 +356,7 @@ object BackupMangaHandler { this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt this[ChapterTable.version] = chapter.version - this[ChapterTable.memo] = Json.decodeFromString(chapter.memo.decodeToString()) + this[ChapterTable.memo] = chapter.memo.decodeToString() }.map { it[ChapterTable.id].value } } else { emptyList() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt index b2a0379b0..c129d33c6 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt @@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceFactory +import eu.kanade.tachiyomi.source.local.LocalSource import io.github.oshai.kotlinlogging.KotlinLogging import net.dongliu.apk.parser.ApkFile import net.dongliu.apk.parser.bean.Icon @@ -23,11 +24,10 @@ import okio.source import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.select import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.extension.ExtensionsList.extensionTableAsDataClass -import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi import suwayomi.tachidesk.manga.impl.util.PackageTools import suwayomi.tachidesk.manga.impl.util.PackageTools.EXTENSION_FEATURE import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX @@ -62,18 +62,20 @@ object Extension { suspend fun installExtension(pkgName: String): Int { logger.debug { "Installing $pkgName" } - val extensionRecord = extensionTableAsDataClass().first { it.pkgName == pkgName } + val apkUrl = + transaction { + ExtensionTable + .select(ExtensionTable.apkUrl) + .where { ExtensionTable.pkgName eq pkgName } + .firstOrNull() + ?.get(ExtensionTable.apkUrl) + } ?: throw NullPointerException("Could not find extension $pkgName") return installAPK { - val apkURL = - ExtensionGithubApi.getApkUrl( - extensionRecord.repo ?: throw NullPointerException("Could not find extension repo"), - extensionRecord.apkName, - ) - val apkName = Uri.parse(apkURL).lastPathSegment!! + val apkName = Uri.parse(apkUrl).lastPathSegment!! val apkSavePath = "${applicationDirs.extensionsRoot}/$apkName" // download apk file - downloadAPKFile(apkURL, apkSavePath) + downloadAPKFile(apkUrl, apkSavePath) apkSavePath } @@ -193,9 +195,9 @@ object Extension { it[name] = extensionName it[this.pkgName] = packageInfo.packageName it[versionName] = packageInfo.versionName - it[versionCode] = packageInfo.versionCode + it[versionCode] = packageInfo.versionCode.toLong() it[lang] = extensionLang - it[this.isNsfw] = isNsfw + it[contentRating] = if (isNsfw) 3 else 0 // todo will change } } @@ -204,7 +206,7 @@ object Extension { it[this.isInstalled] = true it[this.classFQName] = className it[versionName] = packageInfo.versionName - it[versionCode] = packageInfo.versionCode + it[versionCode] = packageInfo.versionCode.toLong() } val extensionId = @@ -220,7 +222,7 @@ object Extension { it[name] = httpSource.name it[lang] = httpSource.lang it[extension] = extensionId - it[SourceTable.isNsfw] = isNsfw + it[contentRating] = if (isNsfw) 3 else 0 } logger.debug { "Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}" } } @@ -343,7 +345,9 @@ object Extension { logger.debug { "Uninstalling $pkgName" } val extensionRecord = transaction { ExtensionTable.selectAll().where { ExtensionTable.pkgName eq pkgName }.first() } - val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk") + val fileNameWithoutType = + extensionRecord[ExtensionTable.apkName]?.substringBefore(".apk") + ?: throw NullPointerException("Missing $pkgName apkName") val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar" val sources = transaction { @@ -359,6 +363,7 @@ object Extension { ExtensionTable.update({ ExtensionTable.pkgName eq pkgName }) { it[isInstalled] = false it[hasUpdate] = false + it[apkName] = null } } @@ -385,8 +390,7 @@ object Extension { it[versionName] = targetExtension.versionName it[versionCode] = targetExtension.versionCode it[lang] = targetExtension.lang - it[isNsfw] = targetExtension.isNsfw - it[apkName] = targetExtension.apkName + it[contentRating] = targetExtension.contentRating.ordinal it[iconUrl] = targetExtension.iconUrl it[hasUpdate] = false } @@ -394,17 +398,21 @@ object Extension { return installExtension(pkgName) } - suspend fun getExtensionIcon(apkName: String): Pair { - val iconUrl = - if (apkName == "localSource") { - "" - } else { - transaction { ExtensionTable.selectAll().where { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl] - } - + suspend fun getExtensionIcon(pkgName: String): Pair { val cacheSaveDir = "${applicationDirs.extensionsRoot}/icon" - return getImageResponse(cacheSaveDir, apkName) { + if (pkgName == LocalSource::class.java.`package`.name) { + return getImageResponse(cacheSaveDir, "localSource") { + network.client + .newCall(GET("", cache = CacheControl.FORCE_NETWORK)) + .await() + } + } + + val iconUrl = + transaction { ExtensionTable.selectAll().where { ExtensionTable.pkgName eq pkgName }.first() }[ExtensionTable.iconUrl] + + return getImageResponse(cacheSaveDir, pkgName) { network.client .newCall( GET(iconUrl, cache = CacheControl.FORCE_NETWORK), @@ -412,5 +420,5 @@ object Extension { } } - fun getExtensionIconUrl(apkName: String): String = "/api/v1/extension/icon/$apkName" + fun proxyExtensionIconUrl(pkgName: String): String = "/api/v1/extension/icon/$pkgName" } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt new file mode 100644 index 000000000..8cdade8a0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionStoreService.kt @@ -0,0 +1,223 @@ +package suwayomi.tachidesk.manga.impl.extension + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.awaitSuccess +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.serialization.decodeFromByteArray +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.okio.decodeFromBufferedSource +import kotlinx.serialization.protobuf.ProtoBuf +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import suwayomi.tachidesk.manga.impl.extension.github.NetworkExtensionStore +import suwayomi.tachidesk.manga.impl.extension.github.NetworkLegacyExtension +import suwayomi.tachidesk.manga.impl.extension.github.NetworkLegacyExtensionRepo +import suwayomi.tachidesk.manga.impl.extension.github.toExtensionInfo +import suwayomi.tachidesk.manga.impl.extension.github.toExtensionInfos +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore +import suwayomi.tachidesk.manga.model.table.ExtensionStoreTable +import suwayomi.tachidesk.server.serverConfig +import uy.kohesive.injekt.injectLazy +import kotlin.coroutines.cancellation.CancellationException + +object ExtensionStoreService { + private val logger = KotlinLogging.logger {} + + val network: NetworkHelper by injectLazy() + val protoBuf: ProtoBuf by injectLazy() + val json: Json by injectLazy() + + suspend fun fetch(indexUrl: String): ExtensionStore = fetch(indexUrl, forceV2 = false) + + private suspend fun fetch( + indexUrl: String, + forceV2: Boolean, + ): ExtensionStore { + var updatedIndexUrl = indexUrl + return try { + val response = network.client.newCall(GET(indexUrl)).awaitSuccess() + response.body + .source() + .use { source -> + try { + protoBuf.decodeFromByteArray(source.peek().readByteArray()) + } catch (e: IllegalArgumentException) { + logger.debug { "Failed to decode as protobuf, trying JSON" } + if (forceV2) throw e + try { + json.decodeFromBufferedSource(source.peek()) + } catch (_: IllegalArgumentException) { + logger.debug { "Failed to decode as NetworkExtensionStore, trying LegacyExtensionRepo" } + val legacyIndex = + try { + json.decodeFromBufferedSource(source.peek()) + } catch (e: IllegalArgumentException) { + if (!indexUrl.endsWith("/index.min.json")) { + throw e + } + logger.debug { "Retrying with /index.min.json" } + updatedIndexUrl = indexUrl.replace("/index.min.json", "/repo.json") + network.client.newCall(GET(updatedIndexUrl)).awaitSuccess().body.source().use { + json.decodeFromBufferedSource(it) + } + } + + if (legacyIndex.indexV2 != null) { + return fetch(legacyIndex.indexV2, forceV2 = true) + } else { + legacyIndex + } + } + } + }.toExtensionStore(updatedIndexUrl) + } catch (e: Exception) { + if (e is CancellationException) throw e + logger.debug(e) { "Failed to fetch extension store '$indexUrl'" } + throw e + } + } + + fun upsert(store: ExtensionStore) { + transaction { + val existing = + ExtensionStoreTable + .selectAll() + .where { ExtensionStoreTable.indexUrl eq store.indexUrl } + .firstOrNull() + + if (existing == null) { + ExtensionStoreTable.insert { + it[name] = store.name + it[badgeLabel] = store.badgeLabel + it[signingKey] = store.signingKey + it[contactWebsite] = store.contact.website + it[contactDiscord] = store.contact.discord + it[indexUrl] = store.indexUrl + it[isLegacy] = store.isLegacy + } + } else { + ExtensionStoreTable.update({ ExtensionStoreTable.indexUrl eq store.indexUrl }) { + it[name] = store.name + it[badgeLabel] = store.badgeLabel + it[signingKey] = store.signingKey + it[contactWebsite] = store.contact.website + it[contactDiscord] = store.contact.discord + it[isLegacy] = store.isLegacy + } + } + } + } + + suspend fun getAndRefresh(): List { + val stores = + transaction { + ExtensionStoreTable.selectAll().toList() + } + return stores.mapNotNull { storeRow -> + val oldIndexUrl = storeRow[ExtensionStoreTable.indexUrl] + val oldName = storeRow[ExtensionStoreTable.name] + try { + val store = fetch(oldIndexUrl) + upsert(store) + if (store.indexUrl != oldIndexUrl) { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl eq oldIndexUrl } + } + syncDbToPrefs() + } + store + } catch (e: Exception) { + logger.warn(e) { "Failed to fetch extension store '$oldName ($oldIndexUrl)'" } + null + } + } + } + + fun syncDbToPrefs() { + val dbStores = + transaction { + ExtensionStoreTable + .selectAll() + .map { it[ExtensionStoreTable.indexUrl] } + .toSet() + } + + val currentPrefs = serverConfig.extensionStores.value.toSet() + val toAdd = dbStores - currentPrefs + val toRemove = currentPrefs - dbStores + + if (toAdd.isNotEmpty()) { + serverConfig.extensionStores.value = (serverConfig.extensionStores.value + toAdd).distinct() + } + + if (toRemove.isNotEmpty()) { + serverConfig.extensionStores.value = serverConfig.extensionStores.value.filterNot { it in toRemove } + } + } + + suspend fun syncPrefsToDb() { + val prefUrls = serverConfig.extensionStores.value.toSet() + + val dbStores = + transaction { + ExtensionStoreTable.selectAll().associateBy { it[ExtensionStoreTable.indexUrl] } + } + + val toAdd = prefUrls - dbStores.keys + + toAdd.forEach { url -> + try { + val store = fetch(url) + upsert(store) + } catch (e: Exception) { + logger.warn(e) { "Failed to sync preference store '$url' to database" } + } + } + + val toRemove = dbStores.keys - prefUrls + if (toRemove.isNotEmpty()) { + transaction { + ExtensionStoreTable.deleteWhere { ExtensionStoreTable.indexUrl inList toRemove.toList() } + } + } + } + + suspend fun getExtensions(store: ExtensionStore): List { + val extensions = + if (!store.isLegacy) { + val response = network.client.newCall(GET(store.indexUrl)).awaitSuccess() + response.body + .source() + .use { source -> + try { + protoBuf.decodeFromByteArray(source.peek().readByteArray()) + } catch (_: IllegalArgumentException) { + json.decodeFromBufferedSource(source.peek()) + } + }.toExtensionInfos(store) + } else { + val storeBaseUrl = store.indexUrl.removeSuffix("/repo.json") + val response = network.client.newCall(GET("$storeBaseUrl/index.min.json")).awaitSuccess() + response.body.source().use { source -> + json + .decodeFromBufferedSource>(source) + .map { it.toExtensionInfo(store, storeBaseUrl) } + } + } + return extensions + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt index 0398f1f5f..1768ff544 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt @@ -21,12 +21,10 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.statements.toExecutable import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update -import suwayomi.tachidesk.manga.impl.extension.Extension.getExtensionIconUrl -import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi -import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension +import suwayomi.tachidesk.manga.impl.extension.Extension.proxyExtensionIconUrl import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo import suwayomi.tachidesk.manga.model.table.ExtensionTable -import suwayomi.tachidesk.server.serverConfig import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration.Companion.seconds @@ -34,23 +32,23 @@ object ExtensionsList { private val logger = KotlinLogging.logger {} var lastUpdateCheck: Long = 0 - var updateMap = ConcurrentHashMap() + var updateMap = ConcurrentHashMap() suspend fun fetchExtensions() { - // update if 60 seconds has passed or requested offline and database is empty - val extensions = - serverConfig.extensionRepos.value.map { repo -> - kotlin - .runCatching { - ExtensionGithubApi.findExtensions(repo.repoUrlReplace()) - }.onFailure { - logger.warn(it) { - "Failed to fetch extensions for repo: $repo" - } - } + val allExtensions = mutableListOf() + + ExtensionStoreService.getAndRefresh().forEach { store -> + try { + val extensions = ExtensionStoreService.getExtensions(store) + allExtensions.addAll(extensions) + } catch (e: Exception) { + logger.warn(e) { + "Failed to fetch extensions for store: ${store.indexUrl}" + } } - val foundExtensions = extensions.mapNotNull { it.getOrNull() }.flatten() - updateExtensionDatabase(foundExtensions) + } + + updateExtensionDatabase(allExtensions) } suspend fun fetchExtensionsCached() { @@ -74,25 +72,25 @@ object ExtensionsList { transaction { ExtensionTable.selectAll().filter { it[ExtensionTable.name] != LocalSource.EXTENSION_NAME }.map { ExtensionDataClass( - it[ExtensionTable.repo], - it[ExtensionTable.apkName], - getExtensionIconUrl(it[ExtensionTable.apkName]), - it[ExtensionTable.name], - it[ExtensionTable.pkgName], - it[ExtensionTable.versionName], - it[ExtensionTable.versionCode], - it[ExtensionTable.lang], - it[ExtensionTable.isNsfw], - it[ExtensionTable.isInstalled], - it[ExtensionTable.hasUpdate], - it[ExtensionTable.isObsolete], + repo = it[ExtensionTable.storeIndexUrl], + apkName = it[ExtensionTable.apkName].orEmpty(), + iconUrl = proxyExtensionIconUrl(it[ExtensionTable.pkgName]), + name = it[ExtensionTable.name], + pkgName = it[ExtensionTable.pkgName], + versionName = it[ExtensionTable.versionName], + versionCode = it[ExtensionTable.versionCode].toInt(), + lang = it[ExtensionTable.lang], + isNsfw = it[ExtensionTable.contentRating] == 3, + installed = it[ExtensionTable.isInstalled], + hasUpdate = it[ExtensionTable.hasUpdate], + obsolete = it[ExtensionTable.isObsolete], ) } } private val updateExtensionDatabaseMutex = Mutex() - private suspend fun updateExtensionDatabase(foundExtensions: List) { + private suspend fun updateExtensionDatabase(foundExtensions: List) { updateExtensionDatabaseMutex.withLock { transaction { val uniqueExtensions = @@ -106,10 +104,10 @@ object ExtensionsList { .selectAll() .toList() .associateBy { it[ExtensionTable.pkgName] } - val extensionsToUpdate = mutableListOf>() - val extensionsToInsert = mutableListOf() + val extensionsToUpdate = mutableListOf>() + val extensionsToInsert = mutableListOf() val extensionsToDelete = - installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) -> + installedExtensions.filter { it.value[ExtensionTable.storeIndexUrl] != null }.mapNotNull { (pkgName, extension) -> extension.takeUnless { uniqueExtensions.any { it.pkgName == pkgName } } } uniqueExtensions.forEach { @@ -132,7 +130,7 @@ object ExtensionsList { addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) // Always update icon url and repo this[ExtensionTable.iconUrl] = foundExtension.iconUrl - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl // add these because batch updates need matching columns this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] @@ -168,13 +166,14 @@ object ExtensionsList { extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) -> addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) // extension is not installed, so we can overwrite the data without a care - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.extensionLib] = foundExtension.extensionLib this[ExtensionTable.versionName] = foundExtension.versionName this[ExtensionTable.versionCode] = foundExtension.versionCode this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.contentRating] = foundExtension.contentRating.ordinal + this[ExtensionTable.apkUrl] = foundExtension.apkUrl this[ExtensionTable.iconUrl] = foundExtension.iconUrl } }.toExecutable() @@ -183,14 +182,15 @@ object ExtensionsList { } if (extensionsToInsert.isNotEmpty()) { ExtensionTable.batchInsert(extensionsToInsert) { foundExtension -> - this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.storeIndexUrl] = foundExtension.storeIndexUrl this[ExtensionTable.name] = foundExtension.name this[ExtensionTable.pkgName] = foundExtension.pkgName + this[ExtensionTable.extensionLib] = foundExtension.extensionLib this[ExtensionTable.versionName] = foundExtension.versionName this[ExtensionTable.versionCode] = foundExtension.versionCode this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.contentRating] = foundExtension.contentRating.ordinal + this[ExtensionTable.apkUrl] = foundExtension.apkUrl this[ExtensionTable.iconUrl] = foundExtension.iconUrl } } @@ -215,16 +215,4 @@ object ExtensionsList { } } } - - private fun String.repoUrlReplace(): String = - if (contains("github")) { - replace(repoMatchRegex) { - "https://raw.githubusercontent.com/${it.groupValues[2]}/${it.groupValues[3]}/" + - (it.groupValues.getOrNull(4)?.ifBlank { null } ?: "repo") + - "/" + - (it.groupValues.getOrNull(5)?.ifBlank { null } ?: "index.min.json") - } - } else { - this - } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt new file mode 100644 index 000000000..68d485216 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/BaseNetworkExtensionStore.kt @@ -0,0 +1,14 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +interface BaseNetworkExtensionStore { + fun toExtensionStore(indexUrl: String): ExtensionStore +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt deleted file mode 100644 index 7da708a1f..000000000 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt +++ /dev/null @@ -1,107 +0,0 @@ -package suwayomi.tachidesk.manga.impl.extension.github - -/* - * Copyright (C) Contributors to the Suwayomi project - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.NetworkHelper -import eu.kanade.tachiyomi.network.awaitSuccess -import eu.kanade.tachiyomi.network.parseAs -import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json -import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX -import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN -import uy.kohesive.injekt.injectLazy - -object ExtensionGithubApi { - private val logger = KotlinLogging.logger {} - private val json: Json by injectLazy() - - @Serializable - private data class ExtensionJsonObject( - val name: String, - val pkg: String, - val apk: String, - val lang: String, - val code: Int, - val version: String, - val nsfw: Int, - val hasReadme: Int = 0, - val hasChangelog: Int = 0, - val sources: List?, - ) - - @Serializable - private data class ExtensionSourceJsonObject( - val name: String, - val lang: String, - val id: Long, - val baseUrl: String, - ) - - suspend fun findExtensions(repo: String): List { - val response = - client.newCall(GET(repo)).awaitSuccess() - - return with(json) { - response - .parseAs>() - .toExtensions(repo.substringBeforeLast('/') + '/') - } - } - - fun getApkUrl( - repo: String, - apkName: String, - ): String = "${repo}apk/$apkName" - - private val client by lazy { - val network: NetworkHelper by injectLazy() - network.client - .newBuilder() - .addNetworkInterceptor { chain -> - val originalResponse = chain.proceed(chain.request()) - originalResponse - .newBuilder() - .header("Content-Type", "application/json") - .build() - }.build() - } - - private fun List.toExtensions(repo: String): List = - this - .filter { - val libVersion = it.version.substringBeforeLast('.').toDouble() - libVersion in LIB_VERSION_MIN..LIB_VERSION_MAX - }.map { - OnlineExtension( - repo = repo, - name = it.name.substringAfter("Tachiyomi: "), - pkgName = it.pkg, - versionName = it.version, - versionCode = it.code, - lang = it.lang, - isNsfw = it.nsfw == 1, - hasReadme = it.hasReadme == 1, - hasChangelog = it.hasChangelog == 1, - sources = it.sources?.toExtensionSources() ?: emptyList(), - apkName = it.apk, - iconUrl = "${repo}icon/${it.pkg}.png", - ) - } - - private fun List.toExtensionSources(): List = - this.map { - OnlineExtensionSource( - name = it.name, - lang = it.lang, - id = it.id, - baseUrl = it.baseUrl, - ) - } -} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt new file mode 100644 index 000000000..8c5aba3ec --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkExtensionStore.kt @@ -0,0 +1,127 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonNames +import kotlinx.serialization.protobuf.ProtoNumber +import suwayomi.tachidesk.manga.model.dataclass.ContentRating +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionSource +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@OptIn(ExperimentalSerializationApi::class) +@Serializable +data class NetworkExtensionStore( + @ProtoNumber(1) val name: String, + @ProtoNumber(2) val badgeLabel: String, + @ProtoNumber(3) val signingKey: String, + @ProtoNumber(4) val contact: Contact, + @ProtoNumber(5) val extensions: List, +) : BaseNetworkExtensionStore { + @Serializable + data class Contact( + @ProtoNumber(1) val website: String, + @ProtoNumber(2) val discord: String?, + ) + + @Serializable + data class Extension( + @ProtoNumber(1) val name: String, + @ProtoNumber(2) val packageName: String, + @ProtoNumber(3) val resources: Resources, + @ProtoNumber(4) val extensionLib: String, + @ProtoNumber(5) val versionCode: Long, + @ProtoNumber(6) val versionName: String, + @ProtoNumber(7) val sources: List, + ) + + @Serializable + data class Resources( + @ProtoNumber(1) val apkUrl: String, + @ProtoNumber(2) val iconUrl: String, + ) + + @Serializable + data class Source( + @ProtoNumber(1) val id: Long, + @ProtoNumber(2) val name: String, + @ProtoNumber(3) val language: String, + @ProtoNumber(4) val homeUrl: String = "", + @ProtoNumber(5) val mirrorUrls: List = emptyList(), + @ProtoNumber(6) val contentRating: ContentRating = ContentRating.SAFE, + @ProtoNumber(7) val message: String? = null, + ) + + @Serializable + enum class ContentRating { + @ProtoNumber(0) + @JsonNames("CONTENT_RATING_SAFE") + SAFE, + + @ProtoNumber(1) + @JsonNames("CONTENT_RATING_SUGGESTIVE") + SUGGESTIVE, + + @ProtoNumber(2) + @JsonNames("CONTENT_RATING_EROTICA") + EROTICA, + + @ProtoNumber(3) + @JsonNames("CONTENT_RATING_PORNOGRAPHIC") + PORNOGRAPHIC, + } + + override fun toExtensionStore(indexUrl: String): ExtensionStore = + ExtensionStore( + indexUrl = indexUrl, + name = name, + badgeLabel = badgeLabel, + signingKey = signingKey, + contact = + ExtensionStore.Contact( + website = contact.website, + discord = contact.discord, + ), + isLegacy = false, + ) +} + +fun NetworkExtensionStore.toExtensionInfos(store: ExtensionStore): List = + extensions.map { extension -> + val lang = extension.sources.map { it.language }.toSet() + ExtensionInfo( + storeIndexUrl = store.indexUrl, + name = extension.name, + pkgName = extension.packageName, + apkUrl = extension.resources.apkUrl, + iconUrl = extension.resources.iconUrl, + extensionLib = extension.extensionLib, + versionCode = extension.versionCode, + versionName = extension.versionName, + lang = if (lang.size == 1) lang.first() else "all", + contentRating = + when (extension.sources.maxOfOrNull { it.contentRating }) { + NetworkExtensionStore.ContentRating.SAFE -> ContentRating.SAFE + NetworkExtensionStore.ContentRating.SUGGESTIVE -> ContentRating.SUGGESTIVE + NetworkExtensionStore.ContentRating.EROTICA -> ContentRating.EROTICA + NetworkExtensionStore.ContentRating.PORNOGRAPHIC -> ContentRating.PORNOGRAPHIC + null -> ContentRating.SAFE + }, + sources = + extension.sources.map { source -> + ExtensionSource( + id = source.id, + name = source.name, + lang = source.language, + homeUrl = source.homeUrl, + ) + }, + ) + } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt new file mode 100644 index 000000000..0a4807452 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtension.kt @@ -0,0 +1,73 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Serializable +import suwayomi.tachidesk.manga.model.dataclass.ContentRating +import suwayomi.tachidesk.manga.model.dataclass.ExtensionInfo +import suwayomi.tachidesk.manga.model.dataclass.ExtensionSource +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@OptIn(ExperimentalSerializationApi::class) +@Serializable +data class NetworkLegacyExtension( + val name: String, + val pkg: String, + val apk: String, + val lang: String, + val version: String, + val code: Long, + val nsfw: Int, + val sources: List? = null, +) { + @Serializable + data class Source( + val id: Long, + val lang: String, + val name: String, + val baseUrl: String, + ) +} + +fun NetworkLegacyExtension.toExtensionInfo( + store: ExtensionStore, + storeBaseUrl: String, +): ExtensionInfo = + ExtensionInfo( + storeIndexUrl = store.indexUrl, + name = name.substringAfter("Tachiyomi: "), + pkgName = pkg, + apkUrl = "$storeBaseUrl/apk/$apk", + iconUrl = "$storeBaseUrl/icon/$pkg.png", + extensionLib = version.substringBeforeLast('.'), + versionCode = code, + versionName = version, + lang = lang, + contentRating = if (nsfw == 1) ContentRating.PORNOGRAPHIC else ContentRating.SAFE, + sources = + if (sources.isNullOrEmpty()) { + listOf( + ExtensionSource( + id = 0, + name = name, + lang = lang, + homeUrl = "", + ), + ) + } else { + sources.map { source -> + ExtensionSource( + id = source.id, + name = source.name, + lang = source.lang, + homeUrl = source.baseUrl, + ) + } + }, + ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt new file mode 100644 index 000000000..c02097dbc --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/NetworkLegacyExtensionRepo.kt @@ -0,0 +1,40 @@ +package suwayomi.tachidesk.manga.impl.extension.github + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import suwayomi.tachidesk.manga.model.dataclass.ExtensionStore + +@Serializable +data class NetworkLegacyExtensionRepo( + @SerialName("index_v2") val indexV2: String?, + val meta: Meta, +) : BaseNetworkExtensionStore { + @Serializable + data class Meta( + val name: String, + val shortName: String?, + val website: String, + val signingKeyFingerprint: String, + ) + + override fun toExtensionStore(indexUrl: String): ExtensionStore = + ExtensionStore( + indexUrl = indexUrl, + name = meta.name, + badgeLabel = meta.shortName ?: meta.name, + signingKey = meta.signingKeyFingerprint, + contact = + ExtensionStore.Contact( + website = meta.website, + discord = null, + ), + isLegacy = true, + ) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt index 9194d3244..1e880c874 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt @@ -41,7 +41,7 @@ object PackageTools { const val METADATA_SOURCE_FACTORY = "tachiyomi.extension.factory" const val METADATA_NSFW = "tachiyomi.extension.nsfw" const val LIB_VERSION_MIN = 1.3 - const val LIB_VERSION_MAX = 1.5 + const val LIB_VERSION_MAX = 1.6 /** * Convert dex to jar, a wrapper for the dex2jar library diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt index 99952debf..2de64c464 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt @@ -45,7 +45,9 @@ object GetCatalogueSource { ExtensionTable.selectAll().where { ExtensionTable.id eq extensionId }.first() } - val apkName = extensionRecord[ExtensionTable.apkName] + val apkName = + extensionRecord[ExtensionTable.apkName] + ?: throw NullPointerException("Missing apkName") val className = extensionRecord[ExtensionTable.classFQName] val jarName = apkName.substringBefore(".apk") + ".jar" val jarPath = "${applicationDirs.extensionsRoot}/$jarName" diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt index c651a5d5d..372bfc230 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/StubSource.kt @@ -29,7 +29,11 @@ open class StubSource( @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPopularManga")) override fun fetchPopularManga(page: Int): Observable = Observable.error(getSourceNotInstalledException()) - override suspend fun getSearchManga(page: Int, query: String, filters: FilterList): MangasPage = throw getSourceNotInstalledException() + override suspend fun getSearchManga( + page: Int, + query: String, + filters: FilterList, + ): MangasPage = throw getSourceNotInstalledException() @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getSearchManga")) override fun fetchSearchManga( @@ -45,7 +49,12 @@ open class StubSource( override fun getFilterList(): FilterList = FilterList() - override suspend fun getMangaUpdate(manga: SManga, chapters: List, fetchDetails: Boolean, fetchChapters: Boolean): SMangaUpdate = throw getSourceNotInstalledException() + override suspend fun getMangaUpdate( + manga: SManga, + chapters: List, + fetchDetails: Boolean, + fetchChapters: Boolean, + ): SMangaUpdate = throw getSourceNotInstalledException() @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getMangaDetails")) override fun fetchMangaDetails(manga: SManga): Observable = Observable.error(getSourceNotInstalledException()) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt similarity index 53% rename from server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt rename to server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt index 8a399b647..357f2de6d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionInfo.kt @@ -1,4 +1,4 @@ -package suwayomi.tachidesk.manga.impl.extension.github +package suwayomi.tachidesk.manga.model.dataclass /* * Copyright (C) Contributors to the Suwayomi project @@ -7,24 +7,30 @@ package suwayomi.tachidesk.manga.impl.extension.github * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -data class OnlineExtensionSource( - val name: String, - val lang: String, - val id: Long, - val baseUrl: String, -) - -data class OnlineExtension( - val repo: String, +data class ExtensionInfo( + val storeIndexUrl: String, val name: String, val pkgName: String, - val apkName: String, - val lang: String, - val versionCode: Int, - val versionName: String, - val isNsfw: Boolean, - val hasReadme: Boolean, - val hasChangelog: Boolean, - val sources: List, + val apkUrl: String, val iconUrl: String, + val extensionLib: String, + val versionCode: Long, + val versionName: String, + val lang: String, + val contentRating: ContentRating, + val sources: List, ) + +data class ExtensionSource( + val id: Long, + val name: String, + val lang: String, + val homeUrl: String, +) + +enum class ContentRating { + SAFE, + SUGGESTIVE, + EROTICA, + PORNOGRAPHIC, +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt new file mode 100644 index 000000000..869f38489 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionStore.kt @@ -0,0 +1,22 @@ +package suwayomi.tachidesk.manga.model.dataclass + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +data class ExtensionStore( + val indexUrl: String, + val name: String, + val badgeLabel: String, + val signingKey: String, + val contact: Contact, + val isLegacy: Boolean, +) { + data class Contact( + val website: String, + val discord: String?, + ) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt index dc1b87d60..c37d0180d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt @@ -7,14 +7,13 @@ package suwayomi.tachidesk.manga.model.table * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ReferenceOption import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.dao.id.IntIdTable -import org.jetbrains.exposed.v1.json.json import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar -import suwayomi.tachidesk.server.database.DBManager +import suwayomi.tachidesk.manga.model.table.columns.unlimitedVarchar object ChapterTable : IntIdTable() { val url = varchar("url", 2048) @@ -46,7 +45,7 @@ object ChapterTable : IntIdTable() { val version = long("version").default(0) val isSyncing = bool("is_syncing").default(false) - val memo = json("memo", DBManager.format) + val memo = unlimitedVarchar("memo") } fun ChapterTable.toDataClass(chapterEntry: ResultRow) = @@ -69,5 +68,5 @@ fun ChapterTable.toDataClass(chapterEntry: ResultRow) = pageCount = chapterEntry[pageCount], lastModifiedAt = chapterEntry[lastModifiedAt], version = chapterEntry[version], - memo = chapterEntry[memo], + memo = Json.decodeFromString(chapterEntry[memo]), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt new file mode 100644 index 000000000..5bba810ef --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionStoreTable.kt @@ -0,0 +1,20 @@ +package suwayomi.tachidesk.manga.model.table + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.v1.core.dao.id.IntIdTable + +object ExtensionStoreTable : IntIdTable() { + val indexUrl = varchar("index_url", 2048) + val name = varchar("name", 256) + val badgeLabel = varchar("badge_label", 32) + val signingKey = varchar("signing_key", 512) + val contactWebsite = varchar("contact_website", 2048) + val contactDiscord = varchar("contact_discord", 2048).nullable() + val isLegacy = bool("is_legacy").default(false) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt index b73cfef24..863647b18 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt @@ -10,8 +10,8 @@ package suwayomi.tachidesk.manga.model.table import org.jetbrains.exposed.v1.core.dao.id.IntIdTable object ExtensionTable : IntIdTable() { - val apkName = varchar("apk_name", 1024) - val repo = varchar("repo", 1024).nullable() + val apkName = varchar("apk_name", 1024).nullable() + val storeIndexUrl = varchar("store_index_url", 2048).nullable() // default is the local source icon from tachiyomi @Suppress("ktlint:standard:max-line-length") @@ -23,10 +23,12 @@ object ExtensionTable : IntIdTable() { val name = varchar("name", 128) val pkgName = varchar("pkg_name", 128) + val apkUrl = varchar("apk_url", 2048) + val extensionLib = varchar("extension_lib", 16).nullable() val versionName = varchar("version_name", 16) - val versionCode = integer("version_code") + val versionCode = long("version_code") val lang = varchar("lang", 32) - val isNsfw = bool("is_nsfw") + val contentRating = integer("content_rating") val isInstalled = bool("is_installed").default(false) val hasUpdate = bool("has_update").default(false) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt index ac49d509a..d4087c841 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt @@ -9,16 +9,14 @@ package suwayomi.tachidesk.manga.model.table import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.UpdateStrategy -import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.dao.id.IntIdTable -import org.jetbrains.exposed.v1.json.json import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass import suwayomi.tachidesk.manga.model.dataclass.toGenreList import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar import suwayomi.tachidesk.manga.model.table.columns.unlimitedVarchar -import suwayomi.tachidesk.server.database.DBManager object MangaTable : IntIdTable() { val url = varchar("url", 2048) @@ -51,7 +49,7 @@ object MangaTable : IntIdTable() { val lastModifiedAt = long("last_modified_at").default(0) val version = long("version").default(0) val isSyncing = bool("is_syncing").default(false) - val memo = json("memo", DBManager.format) + val memo = unlimitedVarchar("memo") } fun MangaTable.toDataClass(mangaEntry: ResultRow) = @@ -76,7 +74,7 @@ fun MangaTable.toDataClass(mangaEntry: ResultRow) = updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]), lastModifiedAt = mangaEntry[lastModifiedAt], version = mangaEntry[version], - memo = mangaEntry[memo], + memo = Json.decodeFromString(mangaEntry[memo]), ) enum class MangaStatus( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt index bdbaa5479..f3bfbe52f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt @@ -8,11 +8,13 @@ package suwayomi.tachidesk.manga.model.table * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import org.jetbrains.exposed.v1.core.dao.id.IdTable +import suwayomi.tachidesk.manga.model.table.columns.unlimitedVarchar object SourceTable : IdTable() { override val id = long("id").entityId() val name = varchar("name", 128) val lang = varchar("lang", 32) val extension = reference("extension", ExtensionTable) - val isNsfw = bool("is_nsfw").default(false) + val message = unlimitedVarchar("message").nullable() + val contentRating = integer("content_rating").default(0) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt index 4bed7a9a9..8d16523ff 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/opds/repository/NavigationRepository.kt @@ -4,7 +4,6 @@ import dev.icerock.moko.resources.StringResource import org.jetbrains.exposed.v1.core.JoinType import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.alias -import org.jetbrains.exposed.v1.core.count import org.jetbrains.exposed.v1.core.countDistinct import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.select @@ -139,9 +138,9 @@ object NavigationRepository { val query = SourceTable .join(ExtensionTable, JoinType.LEFT, onColumn = SourceTable.extension, otherColumn = ExtensionTable.id) - .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .where { ExtensionTable.isInstalled eq true } - .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .orderBy(SourceTable.name to SortOrder.ASC) val totalCount = query.count() @@ -153,7 +152,7 @@ object NavigationRepository { OpdsSourceNavEntry( id = it[SourceTable.id].value, name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]), - iconUrl = it[ExtensionTable.apkName].let { apkName -> Extension.getExtensionIconUrl(apkName) }, + iconUrl = it[ExtensionTable.pkgName].let { pkgName -> Extension.proxyExtensionIconUrl(pkgName) }, mangaCount = null, ) } @@ -173,13 +172,13 @@ object NavigationRepository { .join(ExtensionTable, JoinType.LEFT, onColumn = SourceTable.extension, otherColumn = ExtensionTable.id) .join(CategoryMangaTable, JoinType.LEFT, MangaTable.id, CategoryMangaTable.manga) .join(ChapterTable, JoinType.LEFT, MangaTable.id, ChapterTable.manga) - .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName, mangaCount) + .select(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName, mangaCount) .where { MangaTable.inLibrary eq true } query.applyOpdsMangaFilter(activeFilters, excludeField = "source_id") query - .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .groupBy(SourceTable.id, SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .orderBy(SourceTable.name to SortOrder.ASC) val totalCount = query.count() @@ -195,7 +194,7 @@ object NavigationRepository { OpdsSourceNavEntry( id = it[SourceTable.id].value, name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]), - iconUrl = it[ExtensionTable.apkName].let { apkName -> Extension.getExtensionIconUrl(apkName) }, + iconUrl = it[ExtensionTable.pkgName].let { pkgName -> Extension.proxyExtensionIconUrl(pkgName) }, mangaCount = it[mangaCount], ) } @@ -206,12 +205,12 @@ object NavigationRepository { transaction { SourceTable .join(ExtensionTable, JoinType.LEFT, onColumn = SourceTable.extension, otherColumn = ExtensionTable.id) - .select(SourceTable.name, SourceTable.lang, ExtensionTable.apkName) + .select(SourceTable.name, SourceTable.lang, ExtensionTable.pkgName) .where { SourceTable.id eq sourceId } .firstOrNull() ?.let { val name = formatSourceName(it[SourceTable.name], it[SourceTable.lang]) - val icon = Extension.getExtensionIconUrl(it[ExtensionTable.apkName]) + val icon = Extension.proxyExtensionIconUrl(it[ExtensionTable.pkgName]) Pair(name, icon) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 41b15cec3..ea77725a4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -41,6 +41,7 @@ import suwayomi.tachidesk.graphql.types.DatabaseType import suwayomi.tachidesk.i18n.LocalizationHelper import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport import suwayomi.tachidesk.manga.impl.download.DownloadManager +import suwayomi.tachidesk.manga.impl.extension.ExtensionStoreService import suwayomi.tachidesk.manga.impl.update.IUpdater import suwayomi.tachidesk.manga.impl.update.Updater import suwayomi.tachidesk.manga.impl.util.lang.renameTo @@ -519,4 +520,12 @@ fun applicationSetup() { GlobalScope.launch { CEFManager.init() } + + serverConfig.subscribeTo( + serverConfig.extensionStores, + { _ -> + ExtensionStoreService.syncPrefsToDb() + }, + ignoreInitialValue = false, + ) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddMangaChapterMemoFields.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddMangaChapterMemoFields.kt deleted file mode 100644 index 3cde94c65..000000000 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddMangaChapterMemoFields.kt +++ /dev/null @@ -1,19 +0,0 @@ -package suwayomi.tachidesk.server.database.migration - -/* - * Copyright (C) Contributors to the Suwayomi project - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -import de.neonew.exposed.migrations.helpers.SQLMigration - -@Suppress("ClassName", "unused") -class M0057_AddMangaChapterMemoFields : SQLMigration() { - override val sql = - """ - ALTER TABLE manga ADD COLUMN memo JSON DEFAULT '{}'; - ALTER TABLE chapter ADD COLUMN memo JSON DEFAULT '{}'; - """.trimIndent() -} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt new file mode 100644 index 000000000..bc468dc59 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0057_AddNewExtensionApiFields.kt @@ -0,0 +1,88 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.SQLMigration +import suwayomi.tachidesk.graphql.types.DatabaseType +import suwayomi.tachidesk.server.database.migration.helpers.MAYBE_TYPE_PREFIX +import suwayomi.tachidesk.server.database.migration.helpers.UNLIMITED_TEXT +import suwayomi.tachidesk.server.database.migration.helpers.toSqlName +import suwayomi.tachidesk.server.serverConfig + +@Suppress("ClassName", "unused") +class M0057_AddNewExtensionApiFields : SQLMigration() { + fun postgresRename(): String = + "ALTER TABLE EXTENSION RENAME COLUMN " + "repo".toSqlName() + " TO " + "store_index_url".toSqlName() + ";" + + fun h2Rename(): String = + "ALTER TABLE EXTENSION ALTER COLUMN " + "repo".toSqlName() + " RENAME TO " + "store_index_url".toSqlName() + ";" + + override val sql by lazy { + """ + ALTER TABLE manga ADD COLUMN memo $UNLIMITED_TEXT DEFAULT '{}' NOT NULL; + ALTER TABLE chapter ADD COLUMN memo $UNLIMITED_TEXT DEFAULT '{}' NOT NULL; + ${ + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> postgresRename() + DatabaseType.H2 -> h2Rename() + } + } + ALTER TABLE EXTENSION ALTER COLUMN store_index_url ${MAYBE_TYPE_PREFIX}VARCHAR(2048); + ALTER TABLE EXTENSION ALTER COLUMN version_code ${MAYBE_TYPE_PREFIX}BIGINT; + ALTER TABLE EXTENSION ALTER COLUMN apk_name DROP NOT NULL; + ${ + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> postgresBackfill() + DatabaseType.H2 -> h2Backfill() + } + } + ALTER TABLE EXTENSION ADD COLUMN apk_url VARCHAR(2048); + ALTER TABLE EXTENSION ADD COLUMN content_rating INTEGER DEFAULT 0; + UPDATE EXTENSION SET content_rating = 3 WHERE is_nsfw = TRUE; + ALTER TABLE EXTENSION DROP COLUMN is_nsfw; + ALTER TABLE SOURCE ADD COLUMN message $UNLIMITED_TEXT; + ALTER TABLE SOURCE ADD COLUMN content_rating INTEGER DEFAULT 0; + UPDATE SOURCE SET content_rating = 3 WHERE is_nsfw = TRUE; + ALTER TABLE SOURCE DROP COLUMN is_nsfw; + + + """.trimIndent() + } + + fun postgresBackfill() = + """ + -- 1. Add the column as nullable to avoid table locks + ALTER TABLE EXTENSION ADD COLUMN extension_lib VARCHAR(16); + -- 2. Backfill existing rows using the first two parts of the version_name (split by dot) + UPDATE EXTENSION + SET extension_lib = CONCAT( + SPLIT_PART(version_name, '.', 1), + '.', + SPLIT_PART(version_name, '.', 2) + ); + -- 3. Enforce the NOT NULL constraint + ALTER TABLE EXTENSION ALTER COLUMN extension_lib SET NOT NULL; + """.trimIndent() + + fun h2Backfill() = + """ + -- 1. Add the column as nullable + ALTER TABLE EXTENSION ADD COLUMN extension_lib VARCHAR(16); + -- 2. Backfill rows by extracting text up to the second dot + UPDATE EXTENSION + SET extension_lib = CASE + -- If there's a second dot (e.g. 1.2.3), grab everything before it + WHEN LOCATE('.', version_name, LOCATE('.', version_name) + 1) > 0 + THEN SUBSTRING(version_name, 1, LOCATE('.', version_name, LOCATE('.', version_name) + 1) - 1) + -- If there's no second dot (e.g. 1.2), keep the original value + ELSE version_name + END; + -- 3. Enforce the NOT NULL constraint + ALTER TABLE EXTENSION ALTER COLUMN extension_lib SET NOT NULL; + """.trimIndent() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt new file mode 100644 index 000000000..13ff8bb1d --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0058_AddExtensionStore.kt @@ -0,0 +1,31 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.AddTableMigration +import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.core.dao.id.IntIdTable + +@Suppress("ClassName", "unused") +class M0058_AddExtensionStore : AddTableMigration() { + private class ExtensionStoreTable : IntIdTable() { + val indexUrl = varchar("index_url", 2048) + val name = varchar("name", 256) + val badgeLabel = varchar("badge_label", 32) + val signingKey = varchar("signing_key", 512) + val contactWebsite = varchar("contact_website", 2048) + val contactDiscord = varchar("contact_discord", 2048).nullable() + val isLegacy = bool("is_legacy").default(false) + } + + override val tables: Array + get() = + arrayOf( + ExtensionStoreTable(), + ) +}