temple/Suwayomi-Server
1
0
Fork 0
mirror of https://github.com/Suwayomi/Suwayomi-Server.git synced 2026-07-12 15:24:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature/optimize backup import (#1270)

Browse Source 
* Optimize restoring manga chapters

* Streamline restoring manga data

* Optimize restoring manga trackers

* Simplify passing manga category restore data

* Properly prevent mangas from getting added to default category

76595233fc never actually worked...

* Extract logic to add manga to categories from gql mutation

* Optimize restoring manga categories

* Optimize restoring categories
This commit is contained in:
schroda
2025-02-16 19:00:26 +01:00
committed by GitHub
parent 36cb899b91
commit 633ea97848
5 changed files with 198 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt
View File

@@ -21,17 +21,18 @@ import kotlinx.coroutines.sync.withLock
import okio.buffer
import okio.gzip
import okio.source
import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.batchInsert
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.insertAndGetId
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.graphql.types.toStatus
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
import suwayomi.tachidesk.manga.impl.backup.models.Chapter
import suwayomi.tachidesk.manga.impl.backup.models.Manga
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
@@ -45,7 +46,6 @@ import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
import suwayomi.tachidesk.manga.model.table.CategoryTable
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.server.database.dbTransaction
@@ -179,26 +179,8 @@ object ProtoBackupImport : ProtoBackupBase() {
restoreAmount = backup.backupManga.size + 1 // +1 for categories
updateRestoreState(id, BackupRestoreState.RestoringCategories(backup.backupManga.size))
// Restore categories
if (backup.backupCategories.isNotEmpty()) {
restoreCategories(backup.backupCategories)
}
val categoryMapping =
transaction {
backup.backupCategories.associate {
val dbCategory =
CategoryTable
.selectAll()
.where { CategoryTable.name eq it.name }
.firstOrNull()
val categoryId =
dbCategory?.let { categoryResultRow ->
categoryResultRow[CategoryTable.id].value
} ?: Category.DEFAULT_CATEGORY_ID
it.order to categoryId
}
}
val categoryMapping = restoreCategories(backup.backupCategories)
// Store source mapping for error messages
sourceMapping = backup.getSourceMap()
@@ -216,7 +198,6 @@ object ProtoBackupImport : ProtoBackupBase() {
restoreManga(
backupManga = manga,
backupCategories = backup.backupCategories,
categoryMapping = categoryMapping,
)
}
@@ -240,20 +221,16 @@ object ProtoBackupImport : ProtoBackupBase() {
return validationResult
}
private fun restoreCategories(backupCategories: List<BackupCategory>) {
val dbCategories = Category.getCategoryList()
private fun restoreCategories(backupCategories: List<BackupCategory>): Map<Int, Int> {
val categoryIds = Category.createCategories(backupCategories.map { it.name })
// Iterate over them and create missing categories
backupCategories.forEach { category ->
if (dbCategories.none { it.name == category.name }) {
Category.createCategory(category.name)
}
return backupCategories.withIndex().associate { (index, backupCategory) ->
backupCategory.order to categoryIds[index]
}
}
private fun restoreManga(
backupManga: BackupManga,
backupCategories: List<BackupCategory>,
categoryMapping: Map<Int, Int>,
) {
val manga = backupManga.getMangaImpl()
@@ -261,8 +238,10 @@ object ProtoBackupImport : ProtoBackupBase() {
val categories = backupManga.categories
val history = backupManga.history
val dbCategoryIds = categories.map { categoryMapping[it]!! }
try {
restoreMangaData(manga, chapters, categories, history, backupManga.tracking, backupCategories, categoryMapping)
restoreMangaData(manga, chapters, dbCategoryIds, history, backupManga.tracking)
} catch (e: Exception) {
val sourceName = sourceMapping[manga.source] ?: manga.source.toString()
errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}")
@@ -273,11 +252,9 @@ object ProtoBackupImport : ProtoBackupBase() {
private fun restoreMangaData(
manga: Manga,
chapters: List<Chapter>,
categories: List<Int>,
categoryIds: List<Int>,
history: List<BackupHistory>,
tracks: List<BackupTracking>,
backupCategories: List<BackupCategory>,
categoryMapping: Map<Int, Int>,
) {
val dbManga =
transaction {
@@ -286,12 +263,13 @@ object ProtoBackupImport : ProtoBackupBase() {
.where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
.firstOrNull()
}
val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW
val mangaId =
if (dbManga == null) { // Manga not in database
transaction {
// insert manga to database
val mangaId =
transaction {
val mangaId =
if (dbManga == null) {
// insert manga to database
MangaTable
.insertAndGetId {
it[url] = manga.url
@@ -313,47 +291,36 @@ object ProtoBackupImport : ProtoBackupBase() {
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
}.value
} else {
val dbMangaId = dbManga[MangaTable.id].value
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
// Merge manga data
MangaTable.update({ MangaTable.id eq dbMangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.update_strategy.name
// insert chapter data
restoreMangaChapterData(mangaId, RestoreMode.NEW, chapters)
it[initialized] = dbManga[initialized] || manga.description != null
// insert categories
restoreMangaCategoryData(mangaId, categories, categoryMapping)
it[inLibrary] = manga.favorite || dbManga[inLibrary]
mangaId
}
} else { // Manga in database
transaction {
val mangaId = dbManga[MangaTable.id].value
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
}
// Merge manga data
MangaTable.update({ MangaTable.id eq mangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.update_strategy.name
it[initialized] = dbManga[initialized] || manga.description != null
it[inLibrary] = manga.favorite || dbManga[inLibrary]
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
dbMangaId
}
// merge chapter data
restoreMangaChapterData(mangaId, RestoreMode.EXISTING, chapters)
// merge chapter data
restoreMangaChapterData(mangaId, restoreMode, chapters)
// merge categories
restoreMangaCategoryData(mangaId, categories, categoryMapping)
// merge categories
restoreMangaCategoryData(mangaId, categoryIds)
mangaId
}
mangaId
}
restoreMangaTrackerData(mangaId, tracks)
@@ -361,80 +328,72 @@ object ProtoBackupImport : ProtoBackupBase() {
// TODO: insert/merge history
}
private fun getMangaChapterToRestoreInfo(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<Chapter>,
): Pair<List<Chapter>, List<Pair<Chapter, ResultRow>>> {
val uniqueChapters = chapters.distinctBy { it.url }
if (restoreMode == RestoreMode.NEW) {
return Pair(uniqueChapters, emptyList())
}
val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] }
val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) }
val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! }
return chaptersToInsert to chaptersToUpdateToDbChapter
}
private fun restoreMangaChapterData(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<Chapter>,
) = dbTransaction {
val uniqueChapters = chapters.distinctBy { it.url }
val chaptersLength = uniqueChapters.size
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
if (restoreMode == RestoreMode.NEW) {
ChapterTable.batchInsert(uniqueChapters) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.date_upload == 0L) {
this[ChapterTable.date_upload] = chapter.date_fetch
} else {
this[ChapterTable.date_upload] = chapter.date_upload
}
this[ChapterTable.chapter_number] = chapter.chapter_number
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersLength - chapter.source_order
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.last_page_read.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = TimeUnit.MILLISECONDS.toSeconds(chapter.date_fetch)
ChapterTable.batchInsert(chaptersToInsert) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.date_upload == 0L) {
this[ChapterTable.date_upload] = chapter.date_fetch
} else {
this[ChapterTable.date_upload] = chapter.date_upload
}
this[ChapterTable.chapter_number] = chapter.chapter_number
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.source_order
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.last_page_read.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = TimeUnit.MILLISECONDS.toSeconds(chapter.date_fetch)
}
// merge chapter data
val dbChapters = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }
uniqueChapters.forEach { chapter ->
val dbChapter = dbChapters.find { it[ChapterTable.url] == chapter.url }
if (dbChapter == null) {
ChapterTable.insert {
it[url] = chapter.url
it[name] = chapter.name
if (chapter.date_upload == 0L) {
it[date_upload] = chapter.date_fetch
} else {
it[date_upload] = chapter.date_upload
}
it[chapter_number] = chapter.chapter_number
it[scanlator] = chapter.scanlator
it[sourceOrder] = chaptersLength - chapter.source_order
it[ChapterTable.manga] = mangaId
it[isRead] = chapter.read
it[lastPageRead] = chapter.last_page_read.coerceAtLeast(0)
it[isBookmarked] = chapter.bookmark
}
} else {
ChapterTable.update({ (ChapterTable.url eq dbChapter[ChapterTable.url]) and (ChapterTable.manga eq mangaId) }) {
it[isRead] = chapter.read || dbChapter[isRead]
it[lastPageRead] = max(chapter.last_page_read, dbChapter[lastPageRead]).coerceAtLeast(0)
it[isBookmarked] = chapter.bookmark || dbChapter[isBookmarked]
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
BatchUpdateStatement(ChapterTable).apply {
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
this[ChapterTable.lastPageRead] =
max(backupChapter.last_page_read, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
}
execute(this@dbTransaction)
}
}
}
private fun restoreMangaCategoryData(
mangaId: Int,
categories: List<Int>,
categoryMapping: Map<Int, Int>,
categoryIds: List<Int>,
) {
categories.forEach { backupCategoryOrder ->
CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
}
CategoryManga.addMangaToCategories(mangaId, categoryIds)
}
private fun restoreMangaTrackerData(
@@ -473,8 +432,8 @@ object ProtoBackupImport : ProtoBackupBase() {
}
}.partition { (it.id ?: -1) > 0 }
existingTracks.forEach(Tracker::updateTrackRecord)
newTracks.forEach(Tracker::insertTrackRecord)
Tracker.updateTrackRecords(existingTracks)
Tracker.insertTrackRecords(newTracks)
}
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)