mirror of
https://github.com/Suwayomi/Suwayomi-Server.git
synced 2026-07-03 10:54:38 -05:00
Feature/cleanup backup logic (#1701)
* Extract global metadata backup logic into BackupGlobalMetaHandler * Extract category backup logic into BackupCategoryHandler * Extract source backup logic into BackupSourceHandler * Extract manga backup logic into BackupMangaHandler
This commit is contained in:
schroda
@@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl.backup.proto
|
|||||||
|
|
||||||
import android.app.Application
|
import android.app.Application
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
import eu.kanade.tachiyomi.source.model.UpdateStrategy
|
|
||||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||||
import kotlinx.coroutines.DelicateCoroutinesApi
|
import kotlinx.coroutines.DelicateCoroutinesApi
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
@@ -20,33 +19,14 @@ import okio.Buffer
|
|||||||
import okio.Sink
|
import okio.Sink
|
||||||
import okio.buffer
|
import okio.buffer
|
||||||
import okio.gzip
|
import okio.gzip
|
||||||
import org.jetbrains.exposed.sql.Query
|
|
||||||
import org.jetbrains.exposed.sql.ResultRow
|
|
||||||
import org.jetbrains.exposed.sql.SortOrder
|
|
||||||
import org.jetbrains.exposed.sql.selectAll
|
|
||||||
import org.jetbrains.exposed.sql.transactions.transaction
|
import org.jetbrains.exposed.sql.transactions.transaction
|
||||||
import suwayomi.tachidesk.global.impl.GlobalMeta
|
|
||||||
import suwayomi.tachidesk.manga.impl.Category
|
|
||||||
import suwayomi.tachidesk.manga.impl.CategoryManga
|
|
||||||
import suwayomi.tachidesk.manga.impl.Chapter
|
|
||||||
import suwayomi.tachidesk.manga.impl.Manga
|
|
||||||
import suwayomi.tachidesk.manga.impl.Source
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
|
|
||||||
import suwayomi.tachidesk.manga.impl.track.Track
|
|
||||||
import suwayomi.tachidesk.manga.model.table.CategoryTable
|
|
||||||
import suwayomi.tachidesk.manga.model.table.ChapterTable
|
|
||||||
import suwayomi.tachidesk.manga.model.table.MangaStatus
|
|
||||||
import suwayomi.tachidesk.manga.model.table.MangaTable
|
|
||||||
import suwayomi.tachidesk.manga.model.table.SourceTable
|
|
||||||
import suwayomi.tachidesk.manga.model.table.toDataClass
|
|
||||||
import suwayomi.tachidesk.server.ApplicationDirs
|
import suwayomi.tachidesk.server.ApplicationDirs
|
||||||
import suwayomi.tachidesk.server.serverConfig
|
import suwayomi.tachidesk.server.serverConfig
|
||||||
import suwayomi.tachidesk.util.HAScheduler
|
import suwayomi.tachidesk.util.HAScheduler
|
||||||
@@ -56,7 +36,6 @@ import uy.kohesive.injekt.injectLazy
|
|||||||
import java.io.File
|
import java.io.File
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import kotlin.time.Duration.Companion.days
|
import kotlin.time.Duration.Companion.days
|
||||||
import kotlin.time.Duration.Companion.seconds
|
|
||||||
|
|
||||||
object ProtoBackupExport : ProtoBackupBase() {
|
object ProtoBackupExport : ProtoBackupBase() {
|
||||||
private val logger = KotlinLogging.logger { }
|
private val logger = KotlinLogging.logger { }
|
||||||
@@ -170,20 +149,15 @@ object ProtoBackupExport : ProtoBackupBase() {
|
|||||||
fun createBackup(flags: BackupFlags): InputStream {
|
fun createBackup(flags: BackupFlags): InputStream {
|
||||||
// Create root object
|
// Create root object
|
||||||
|
|
||||||
val databaseManga =
|
val backupMangas = BackupMangaHandler.backup(flags)
|
||||||
if (flags.includeManga) {
|
|
||||||
transaction { MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList() }
|
|
||||||
} else {
|
|
||||||
emptyList()
|
|
||||||
}
|
|
||||||
|
|
||||||
val backup: Backup =
|
val backup: Backup =
|
||||||
transaction {
|
transaction {
|
||||||
Backup(
|
Backup(
|
||||||
backupManga(databaseManga, flags),
|
BackupMangaHandler.backup(flags),
|
||||||
backupCategories(flags),
|
BackupCategoryHandler.backup(flags),
|
||||||
backupExtensionInfo(databaseManga, flags),
|
BackupSourceHandler.backup(backupMangas, flags),
|
||||||
backupGlobalMeta(flags),
|
BackupGlobalMetaHandler.backup(flags),
|
||||||
BackupSettingsHandler.backup(flags),
|
BackupSettingsHandler.backup(flags),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -198,171 +172,4 @@ object ProtoBackupExport : ProtoBackupBase() {
|
|||||||
|
|
||||||
return byteStream.inputStream()
|
return byteStream.inputStream()
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun backupManga(
|
|
||||||
databaseManga: List<ResultRow>,
|
|
||||||
flags: BackupFlags,
|
|
||||||
): List<BackupManga> =
|
|
||||||
databaseManga.map { mangaRow ->
|
|
||||||
val backupManga =
|
|
||||||
BackupManga(
|
|
||||||
source = mangaRow[MangaTable.sourceReference],
|
|
||||||
url = mangaRow[MangaTable.url],
|
|
||||||
title = mangaRow[MangaTable.title],
|
|
||||||
artist = mangaRow[MangaTable.artist],
|
|
||||||
author = mangaRow[MangaTable.author],
|
|
||||||
description = mangaRow[MangaTable.description],
|
|
||||||
genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
|
|
||||||
status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
|
|
||||||
thumbnailUrl = mangaRow[MangaTable.thumbnail_url],
|
|
||||||
dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
|
|
||||||
viewer = 0, // not supported in Tachidesk
|
|
||||||
updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
|
|
||||||
)
|
|
||||||
|
|
||||||
val mangaId = mangaRow[MangaTable.id].value
|
|
||||||
|
|
||||||
if (flags.includeClientData) {
|
|
||||||
backupManga.meta = Manga.getMangaMetaMap(mangaId)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeChapters || flags.includeHistory) {
|
|
||||||
val chapters =
|
|
||||||
transaction {
|
|
||||||
ChapterTable
|
|
||||||
.selectAll()
|
|
||||||
.where { ChapterTable.manga eq mangaId }
|
|
||||||
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
|
|
||||||
.map {
|
|
||||||
ChapterTable.toDataClass(it)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (flags.includeChapters) {
|
|
||||||
val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id })
|
|
||||||
|
|
||||||
backupManga.chapters =
|
|
||||||
chapters.map {
|
|
||||||
BackupChapter(
|
|
||||||
it.url,
|
|
||||||
it.name,
|
|
||||||
it.scanlator,
|
|
||||||
it.read,
|
|
||||||
it.bookmarked,
|
|
||||||
it.lastPageRead,
|
|
||||||
it.fetchedAt.seconds.inWholeMilliseconds,
|
|
||||||
it.uploadDate,
|
|
||||||
it.chapterNumber,
|
|
||||||
chapters.size - it.index,
|
|
||||||
).apply {
|
|
||||||
if (flags.includeClientData) {
|
|
||||||
this.meta = chapterToMeta[it.id] ?: emptyMap()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (flags.includeHistory) {
|
|
||||||
backupManga.history =
|
|
||||||
chapters.mapNotNull {
|
|
||||||
if (it.lastReadAt > 0) {
|
|
||||||
BackupHistory(
|
|
||||||
url = it.url,
|
|
||||||
lastRead = it.lastReadAt.seconds.inWholeMilliseconds,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeCategories) {
|
|
||||||
backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeTracking) {
|
|
||||||
val tracks =
|
|
||||||
Track.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
|
|
||||||
if (it.record == null) {
|
|
||||||
null
|
|
||||||
} else {
|
|
||||||
BackupTracking(
|
|
||||||
syncId = it.record.trackerId,
|
|
||||||
// forced not null so its compatible with 1.x backup system
|
|
||||||
libraryId = it.record.libraryId ?: 0,
|
|
||||||
mediaId = it.record.remoteId,
|
|
||||||
title = it.record.title,
|
|
||||||
lastChapterRead = it.record.lastChapterRead.toFloat(),
|
|
||||||
totalChapters = it.record.totalChapters,
|
|
||||||
score = it.record.score.toFloat(),
|
|
||||||
status = it.record.status,
|
|
||||||
startedReadingDate = it.record.startDate,
|
|
||||||
finishedReadingDate = it.record.finishDate,
|
|
||||||
trackingUrl = it.record.remoteUrl,
|
|
||||||
private = it.record.private,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (tracks.isNotEmpty()) {
|
|
||||||
backupManga.tracking = tracks
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
backupManga
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun backupCategories(flags: BackupFlags): List<BackupCategory> {
|
|
||||||
val categories =
|
|
||||||
CategoryTable
|
|
||||||
.selectAll()
|
|
||||||
.orderBy(CategoryTable.order to SortOrder.ASC)
|
|
||||||
.map { CategoryTable.toDataClass(it) }
|
|
||||||
val categoryToMeta = Category.getCategoriesMetaMaps(categories.map { it.id })
|
|
||||||
|
|
||||||
return categories.map {
|
|
||||||
BackupCategory(
|
|
||||||
it.name,
|
|
||||||
it.order,
|
|
||||||
0, // not supported in Tachidesk
|
|
||||||
).apply {
|
|
||||||
if (flags.includeClientData) {
|
|
||||||
this.meta = categoryToMeta[it.id] ?: emptyMap()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun backupExtensionInfo(
|
|
||||||
mangas: List<ResultRow>,
|
|
||||||
flags: BackupFlags,
|
|
||||||
): List<BackupSource> {
|
|
||||||
val inLibraryMangaSourceIds =
|
|
||||||
mangas
|
|
||||||
.asSequence()
|
|
||||||
.map { it[MangaTable.sourceReference] }
|
|
||||||
.distinct()
|
|
||||||
.toList()
|
|
||||||
val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds }
|
|
||||||
val sourceToMeta = Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value })
|
|
||||||
|
|
||||||
return inLibraryMangaSourceIds
|
|
||||||
.map { mangaSourceId ->
|
|
||||||
val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId }
|
|
||||||
BackupSource(
|
|
||||||
source?.get(SourceTable.name) ?: "",
|
|
||||||
mangaSourceId,
|
|
||||||
).apply {
|
|
||||||
if (flags.includeClientData) {
|
|
||||||
this.meta = sourceToMeta[mangaSourceId] ?: emptyMap()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}.toList()
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun backupGlobalMeta(flags: BackupFlags): Map<String, String> {
|
|
||||||
if (!flags.includeClientData) {
|
|
||||||
return emptyMap()
|
|
||||||
}
|
|
||||||
|
|
||||||
return GlobalMeta.getMetaMap()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,50 +21,21 @@ import kotlinx.coroutines.sync.withLock
|
|||||||
import okio.buffer
|
import okio.buffer
|
||||||
import okio.gzip
|
import okio.gzip
|
||||||
import okio.source
|
import okio.source
|
||||||
import org.jetbrains.exposed.dao.id.EntityID
|
|
||||||
import org.jetbrains.exposed.sql.ResultRow
|
|
||||||
import org.jetbrains.exposed.sql.and
|
|
||||||
import org.jetbrains.exposed.sql.batchInsert
|
|
||||||
import org.jetbrains.exposed.sql.insertAndGetId
|
|
||||||
import org.jetbrains.exposed.sql.selectAll
|
|
||||||
import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
|
|
||||||
import org.jetbrains.exposed.sql.transactions.transaction
|
|
||||||
import org.jetbrains.exposed.sql.update
|
|
||||||
import suwayomi.tachidesk.global.impl.GlobalMeta
|
|
||||||
import suwayomi.tachidesk.graphql.types.toStatus
|
import suwayomi.tachidesk.graphql.types.toStatus
|
||||||
import suwayomi.tachidesk.manga.impl.Category
|
|
||||||
import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
|
|
||||||
import suwayomi.tachidesk.manga.impl.CategoryManga
|
|
||||||
import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas
|
|
||||||
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
|
|
||||||
import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas
|
|
||||||
import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
|
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
|
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
|
|
||||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
|
|
||||||
import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
|
|
||||||
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
|
|
||||||
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
|
|
||||||
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
|
|
||||||
import suwayomi.tachidesk.manga.model.table.ChapterTable
|
|
||||||
import suwayomi.tachidesk.manga.model.table.MangaTable
|
|
||||||
import suwayomi.tachidesk.server.database.dbTransaction
|
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.util.Date
|
import java.util.Date
|
||||||
import java.util.Timer
|
import java.util.Timer
|
||||||
import java.util.TimerTask
|
import java.util.TimerTask
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import kotlin.math.max
|
|
||||||
import kotlin.time.Duration.Companion.milliseconds
|
|
||||||
import suwayomi.tachidesk.manga.impl.track.Track as Tracker
|
|
||||||
|
|
||||||
object ProtoBackupImport : ProtoBackupBase() {
|
object ProtoBackupImport : ProtoBackupBase() {
|
||||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||||
@@ -73,11 +44,6 @@ object ProtoBackupImport : ProtoBackupBase() {
|
|||||||
|
|
||||||
private val backupMutex = Mutex()
|
private val backupMutex = Mutex()
|
||||||
|
|
||||||
enum class RestoreMode {
|
|
||||||
NEW,
|
|
||||||
EXISTING,
|
|
||||||
}
|
|
||||||
|
|
||||||
sealed class BackupRestoreState {
|
sealed class BackupRestoreState {
|
||||||
data object Idle : BackupRestoreState()
|
data object Idle : BackupRestoreState()
|
||||||
|
|
||||||
@@ -215,7 +181,7 @@ object ProtoBackupImport : ProtoBackupBase() {
|
|||||||
val categoryMapping =
|
val categoryMapping =
|
||||||
if (flags.includeCategories) {
|
if (flags.includeCategories) {
|
||||||
updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount))
|
updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount))
|
||||||
restoreCategories(backup.backupCategories)
|
BackupCategoryHandler.restore(backup.backupCategories)
|
||||||
} else {
|
} else {
|
||||||
emptyMap()
|
emptyMap()
|
||||||
}
|
}
|
||||||
@@ -223,9 +189,9 @@ object ProtoBackupImport : ProtoBackupBase() {
|
|||||||
if (flags.includeClientData) {
|
if (flags.includeClientData) {
|
||||||
updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount))
|
updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount))
|
||||||
|
|
||||||
restoreGlobalMeta(backup.meta)
|
BackupGlobalMetaHandler.restore(backup.meta)
|
||||||
|
|
||||||
restoreSourceMeta(backup.backupSources)
|
BackupSourceHandler.restore(backup.backupSources)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store source mapping for error messages
|
// Store source mapping for error messages
|
||||||
@@ -245,7 +211,7 @@ object ProtoBackupImport : ProtoBackupBase() {
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
restoreManga(
|
BackupMangaHandler.restore(
|
||||||
backupManga = manga,
|
backupManga = manga,
|
||||||
categoryMapping = categoryMapping,
|
categoryMapping = categoryMapping,
|
||||||
sourceMapping = sourceMapping,
|
sourceMapping = sourceMapping,
|
||||||
@@ -273,292 +239,4 @@ object ProtoBackupImport : ProtoBackupBase() {
|
|||||||
|
|
||||||
return validationResult
|
return validationResult
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun restoreCategories(backupCategories: List<BackupCategory>): Map<Int, Int> {
|
|
||||||
val categoryIds = Category.createCategories(backupCategories.map { it.name })
|
|
||||||
|
|
||||||
val metaEntryByCategoryId =
|
|
||||||
categoryIds
|
|
||||||
.zip(backupCategories)
|
|
||||||
.associate { (categoryId, backupCategory) ->
|
|
||||||
categoryId to backupCategory.meta
|
|
||||||
}
|
|
||||||
|
|
||||||
modifyCategoriesMetas(metaEntryByCategoryId)
|
|
||||||
|
|
||||||
return backupCategories.withIndex().associate { (index, backupCategory) ->
|
|
||||||
backupCategory.order to categoryIds[index]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreManga(
|
|
||||||
backupManga: BackupManga,
|
|
||||||
categoryMapping: Map<Int, Int>,
|
|
||||||
sourceMapping: Map<Long, String>,
|
|
||||||
errors: MutableList<Pair<Date, String>>,
|
|
||||||
flags: BackupFlags,
|
|
||||||
) {
|
|
||||||
val chapters = backupManga.chapters
|
|
||||||
val categories = backupManga.categories
|
|
||||||
val history = backupManga.history
|
|
||||||
val tracking = backupManga.tracking
|
|
||||||
|
|
||||||
val dbCategoryIds = categories.mapNotNull { categoryMapping[it] }
|
|
||||||
|
|
||||||
try {
|
|
||||||
restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags)
|
|
||||||
} catch (e: Exception) {
|
|
||||||
val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString()
|
|
||||||
errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreMangaData(
|
|
||||||
manga: BackupManga,
|
|
||||||
chapters: List<BackupChapter>,
|
|
||||||
categoryIds: List<Int>,
|
|
||||||
history: List<BackupHistory>,
|
|
||||||
tracks: List<BackupTracking>,
|
|
||||||
flags: BackupFlags,
|
|
||||||
) {
|
|
||||||
val dbManga =
|
|
||||||
transaction {
|
|
||||||
MangaTable
|
|
||||||
.selectAll()
|
|
||||||
.where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
|
|
||||||
.firstOrNull()
|
|
||||||
}
|
|
||||||
val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW
|
|
||||||
|
|
||||||
val mangaId =
|
|
||||||
transaction {
|
|
||||||
val mangaId =
|
|
||||||
if (dbManga == null) {
|
|
||||||
// insert manga to database
|
|
||||||
MangaTable
|
|
||||||
.insertAndGetId {
|
|
||||||
it[url] = manga.url
|
|
||||||
it[title] = manga.title
|
|
||||||
|
|
||||||
it[artist] = manga.artist
|
|
||||||
it[author] = manga.author
|
|
||||||
it[description] = manga.description
|
|
||||||
it[genre] = manga.genre.joinToString()
|
|
||||||
it[status] = manga.status
|
|
||||||
it[thumbnail_url] = manga.thumbnailUrl
|
|
||||||
it[updateStrategy] = manga.updateStrategy.name
|
|
||||||
|
|
||||||
it[sourceReference] = manga.source
|
|
||||||
|
|
||||||
it[initialized] = manga.description != null
|
|
||||||
|
|
||||||
it[inLibrary] = manga.favorite
|
|
||||||
|
|
||||||
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
|
|
||||||
}.value
|
|
||||||
} else {
|
|
||||||
val dbMangaId = dbManga[MangaTable.id].value
|
|
||||||
|
|
||||||
// Merge manga data
|
|
||||||
MangaTable.update({ MangaTable.id eq dbMangaId }) {
|
|
||||||
it[artist] = manga.artist ?: dbManga[artist]
|
|
||||||
it[author] = manga.author ?: dbManga[author]
|
|
||||||
it[description] = manga.description ?: dbManga[description]
|
|
||||||
it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre]
|
|
||||||
it[status] = manga.status
|
|
||||||
it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url]
|
|
||||||
it[updateStrategy] = manga.updateStrategy.name
|
|
||||||
|
|
||||||
it[initialized] = dbManga[initialized] || manga.description != null
|
|
||||||
|
|
||||||
it[inLibrary] = manga.favorite || dbManga[inLibrary]
|
|
||||||
|
|
||||||
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
|
|
||||||
}
|
|
||||||
|
|
||||||
dbMangaId
|
|
||||||
}
|
|
||||||
|
|
||||||
// delete thumbnail in case cached data still exists
|
|
||||||
clearThumbnail(mangaId)
|
|
||||||
|
|
||||||
if (flags.includeClientData && manga.meta.isNotEmpty()) {
|
|
||||||
modifyMangasMetas(mapOf(mangaId to manga.meta))
|
|
||||||
}
|
|
||||||
|
|
||||||
// merge chapter data
|
|
||||||
if (flags.includeChapters || flags.includeHistory) {
|
|
||||||
restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
|
|
||||||
}
|
|
||||||
|
|
||||||
// merge categories
|
|
||||||
if (flags.includeCategories) {
|
|
||||||
restoreMangaCategoryData(mangaId, categoryIds)
|
|
||||||
}
|
|
||||||
|
|
||||||
mangaId
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeTracking) {
|
|
||||||
restoreMangaTrackerData(mangaId, tracks)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: insert/merge history
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun getMangaChapterToRestoreInfo(
|
|
||||||
mangaId: Int,
|
|
||||||
restoreMode: RestoreMode,
|
|
||||||
chapters: List<BackupChapter>,
|
|
||||||
): Pair<List<BackupChapter>, List<Pair<BackupChapter, ResultRow>>> {
|
|
||||||
val uniqueChapters = chapters.distinctBy { it.url }
|
|
||||||
|
|
||||||
if (restoreMode == RestoreMode.NEW) {
|
|
||||||
return Pair(uniqueChapters, emptyList())
|
|
||||||
}
|
|
||||||
|
|
||||||
val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] }
|
|
||||||
|
|
||||||
val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) }
|
|
||||||
val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! }
|
|
||||||
|
|
||||||
return chaptersToInsert to chaptersToUpdateToDbChapter
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreMangaChapterData(
|
|
||||||
mangaId: Int,
|
|
||||||
restoreMode: RestoreMode,
|
|
||||||
chapters: List<BackupChapter>,
|
|
||||||
history: List<BackupHistory>,
|
|
||||||
flags: BackupFlags,
|
|
||||||
) = dbTransaction {
|
|
||||||
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
|
|
||||||
val historyByChapter = history.groupBy({ it.url }, { it.lastRead })
|
|
||||||
|
|
||||||
val insertedChapterIds =
|
|
||||||
if (flags.includeChapters) {
|
|
||||||
ChapterTable
|
|
||||||
.batchInsert(chaptersToInsert) { chapter ->
|
|
||||||
this[ChapterTable.url] = chapter.url
|
|
||||||
this[ChapterTable.name] = chapter.name
|
|
||||||
if (chapter.dateUpload == 0L) {
|
|
||||||
this[ChapterTable.date_upload] = chapter.dateFetch
|
|
||||||
} else {
|
|
||||||
this[ChapterTable.date_upload] = chapter.dateUpload
|
|
||||||
}
|
|
||||||
this[ChapterTable.chapter_number] = chapter.chapterNumber
|
|
||||||
this[ChapterTable.scanlator] = chapter.scanlator
|
|
||||||
|
|
||||||
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
|
|
||||||
this[ChapterTable.manga] = mangaId
|
|
||||||
|
|
||||||
this[ChapterTable.isRead] = chapter.read
|
|
||||||
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
|
|
||||||
this[ChapterTable.isBookmarked] = chapter.bookmark
|
|
||||||
|
|
||||||
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
|
|
||||||
|
|
||||||
if (flags.includeHistory) {
|
|
||||||
this[ChapterTable.lastReadAt] =
|
|
||||||
historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
|
|
||||||
}
|
|
||||||
}.map { it[ChapterTable.id].value }
|
|
||||||
} else {
|
|
||||||
emptyList()
|
|
||||||
}
|
|
||||||
|
|
||||||
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
|
|
||||||
BatchUpdateStatement(ChapterTable).apply {
|
|
||||||
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
|
|
||||||
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
|
|
||||||
if (flags.includeChapters) {
|
|
||||||
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
|
|
||||||
this[ChapterTable.lastPageRead] =
|
|
||||||
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
|
|
||||||
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeHistory) {
|
|
||||||
this[ChapterTable.lastReadAt] =
|
|
||||||
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
|
|
||||||
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
execute(this@dbTransaction)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.includeClientData) {
|
|
||||||
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
|
|
||||||
val chapterToUpdateByChapterId =
|
|
||||||
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
|
|
||||||
dbChapter[ChapterTable.id].value to
|
|
||||||
backupChapter
|
|
||||||
}
|
|
||||||
val metaEntryByChapterId =
|
|
||||||
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
|
|
||||||
.associate { (chapterId, backupChapter) ->
|
|
||||||
chapterId to backupChapter.meta
|
|
||||||
}
|
|
||||||
|
|
||||||
modifyChaptersMetas(metaEntryByChapterId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreMangaCategoryData(
|
|
||||||
mangaId: Int,
|
|
||||||
categoryIds: List<Int>,
|
|
||||||
) {
|
|
||||||
CategoryManga.addMangaToCategories(mangaId, categoryIds)
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreMangaTrackerData(
|
|
||||||
mangaId: Int,
|
|
||||||
tracks: List<BackupTracking>,
|
|
||||||
) {
|
|
||||||
val dbTrackRecordsByTrackerId =
|
|
||||||
Tracker
|
|
||||||
.getTrackRecordsByMangaId(mangaId)
|
|
||||||
.mapNotNull { it.record?.toTrack() }
|
|
||||||
.associateBy { it.tracker_id }
|
|
||||||
|
|
||||||
val (existingTracks, newTracks) =
|
|
||||||
tracks
|
|
||||||
.mapNotNull { backupTrack ->
|
|
||||||
val track = backupTrack.toTrack(mangaId)
|
|
||||||
|
|
||||||
val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null
|
|
||||||
if (isUnsupportedTracker) {
|
|
||||||
return@mapNotNull null
|
|
||||||
}
|
|
||||||
|
|
||||||
val dbTrack =
|
|
||||||
dbTrackRecordsByTrackerId[backupTrack.syncId]
|
|
||||||
?: // new track
|
|
||||||
return@mapNotNull track
|
|
||||||
|
|
||||||
if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
|
|
||||||
return@mapNotNull null
|
|
||||||
}
|
|
||||||
|
|
||||||
dbTrack.also {
|
|
||||||
it.remote_id = track.remote_id
|
|
||||||
it.library_id = track.library_id
|
|
||||||
it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
|
|
||||||
}
|
|
||||||
}.partition { (it.id ?: -1) > 0 }
|
|
||||||
|
|
||||||
Tracker.updateTrackRecords(existingTracks)
|
|
||||||
Tracker.insertTrackRecords(newTracks)
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreGlobalMeta(meta: Map<String, String>) {
|
|
||||||
GlobalMeta.modifyMetas(meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun restoreSourceMeta(backupSources: List<BackupSource>) {
|
|
||||||
modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta })
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright (C) Contributors to the Suwayomi project
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||||
|
|
||||||
|
import org.jetbrains.exposed.sql.SortOrder
|
||||||
|
import org.jetbrains.exposed.sql.selectAll
|
||||||
|
import suwayomi.tachidesk.manga.impl.Category
|
||||||
|
import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
|
||||||
|
import suwayomi.tachidesk.manga.model.table.CategoryTable
|
||||||
|
import suwayomi.tachidesk.manga.model.table.toDataClass
|
||||||
|
import suwayomi.tachidesk.server.database.dbTransaction
|
||||||
|
|
||||||
|
object BackupCategoryHandler {
|
||||||
|
fun backup(flags: BackupFlags): List<BackupCategory> =
|
||||||
|
dbTransaction {
|
||||||
|
val categories =
|
||||||
|
CategoryTable
|
||||||
|
.selectAll()
|
||||||
|
.orderBy(CategoryTable.order to SortOrder.ASC)
|
||||||
|
.map { CategoryTable.toDataClass(it) }
|
||||||
|
|
||||||
|
val categoryToMeta =
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
Category.getCategoriesMetaMaps(categories.map { it.id })
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
categories.map {
|
||||||
|
BackupCategory(
|
||||||
|
it.name,
|
||||||
|
it.order,
|
||||||
|
0, // not supported in Tachidesk
|
||||||
|
).apply {
|
||||||
|
this.meta = categoryToMeta[it.id] ?: emptyMap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun restore(backupCategories: List<BackupCategory>): Map<Int, Int> {
|
||||||
|
val categoryIds = Category.createCategories(backupCategories.map { it.name })
|
||||||
|
|
||||||
|
val metaEntryByCategoryId =
|
||||||
|
categoryIds
|
||||||
|
.zip(backupCategories)
|
||||||
|
.associate { (categoryId, backupCategory) ->
|
||||||
|
categoryId to backupCategory.meta
|
||||||
|
}
|
||||||
|
|
||||||
|
modifyCategoriesMetas(metaEntryByCategoryId)
|
||||||
|
|
||||||
|
return backupCategories.withIndex().associate { (index, backupCategory) ->
|
||||||
|
backupCategory.order to categoryIds[index]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright (C) Contributors to the Suwayomi project
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||||
|
|
||||||
|
import suwayomi.tachidesk.global.impl.GlobalMeta
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
|
|
||||||
|
object BackupGlobalMetaHandler {
|
||||||
|
fun backup(flags: BackupFlags): Map<String, String> {
|
||||||
|
if (!flags.includeClientData) {
|
||||||
|
return emptyMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
return GlobalMeta.getMetaMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun restore(meta: Map<String, String>) {
|
||||||
|
GlobalMeta.modifyMetas(meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,431 @@
|
|||||||
|
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright (C) Contributors to the Suwayomi project
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||||
|
|
||||||
|
import eu.kanade.tachiyomi.source.model.UpdateStrategy
|
||||||
|
import org.jetbrains.exposed.dao.id.EntityID
|
||||||
|
import org.jetbrains.exposed.sql.ResultRow
|
||||||
|
import org.jetbrains.exposed.sql.SortOrder
|
||||||
|
import org.jetbrains.exposed.sql.and
|
||||||
|
import org.jetbrains.exposed.sql.batchInsert
|
||||||
|
import org.jetbrains.exposed.sql.insertAndGetId
|
||||||
|
import org.jetbrains.exposed.sql.selectAll
|
||||||
|
import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
|
||||||
|
import org.jetbrains.exposed.sql.transactions.transaction
|
||||||
|
import org.jetbrains.exposed.sql.update
|
||||||
|
import suwayomi.tachidesk.manga.impl.CategoryManga
|
||||||
|
import suwayomi.tachidesk.manga.impl.Chapter
|
||||||
|
import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas
|
||||||
|
import suwayomi.tachidesk.manga.impl.Manga
|
||||||
|
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
|
||||||
|
import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
|
||||||
|
import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
|
||||||
|
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
|
||||||
|
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
|
||||||
|
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
|
||||||
|
import suwayomi.tachidesk.manga.model.table.ChapterTable
|
||||||
|
import suwayomi.tachidesk.manga.model.table.MangaStatus
|
||||||
|
import suwayomi.tachidesk.manga.model.table.MangaTable
|
||||||
|
import suwayomi.tachidesk.manga.model.table.toDataClass
|
||||||
|
import suwayomi.tachidesk.server.database.dbTransaction
|
||||||
|
import java.util.Date
|
||||||
|
import kotlin.math.max
|
||||||
|
import kotlin.time.Duration.Companion.milliseconds
|
||||||
|
import kotlin.time.Duration.Companion.seconds
|
||||||
|
import suwayomi.tachidesk.manga.impl.track.Track as Tracker
|
||||||
|
|
||||||
|
object BackupMangaHandler {
|
||||||
|
private enum class RestoreMode {
|
||||||
|
NEW,
|
||||||
|
EXISTING,
|
||||||
|
}
|
||||||
|
|
||||||
|
fun backup(flags: BackupFlags): List<BackupManga> =
|
||||||
|
dbTransaction {
|
||||||
|
if (!flags.includeManga) {
|
||||||
|
return@dbTransaction emptyList()
|
||||||
|
}
|
||||||
|
|
||||||
|
val manga = MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList()
|
||||||
|
|
||||||
|
manga.map { mangaRow ->
|
||||||
|
val backupManga =
|
||||||
|
BackupManga(
|
||||||
|
source = mangaRow[MangaTable.sourceReference],
|
||||||
|
url = mangaRow[MangaTable.url],
|
||||||
|
title = mangaRow[MangaTable.title],
|
||||||
|
artist = mangaRow[MangaTable.artist],
|
||||||
|
author = mangaRow[MangaTable.author],
|
||||||
|
description = mangaRow[MangaTable.description],
|
||||||
|
genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
|
||||||
|
status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
|
||||||
|
thumbnailUrl = mangaRow[MangaTable.thumbnail_url],
|
||||||
|
dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
|
||||||
|
viewer = 0, // not supported in Tachidesk
|
||||||
|
updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
|
||||||
|
)
|
||||||
|
|
||||||
|
val mangaId = mangaRow[MangaTable.id].value
|
||||||
|
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
backupManga.meta = Manga.getMangaMetaMap(mangaId)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeChapters || flags.includeHistory) {
|
||||||
|
val chapters =
|
||||||
|
transaction {
|
||||||
|
ChapterTable
|
||||||
|
.selectAll()
|
||||||
|
.where { ChapterTable.manga eq mangaId }
|
||||||
|
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
|
||||||
|
.map {
|
||||||
|
ChapterTable.toDataClass(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (flags.includeChapters) {
|
||||||
|
val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id })
|
||||||
|
|
||||||
|
backupManga.chapters =
|
||||||
|
chapters.map {
|
||||||
|
BackupChapter(
|
||||||
|
it.url,
|
||||||
|
it.name,
|
||||||
|
it.scanlator,
|
||||||
|
it.read,
|
||||||
|
it.bookmarked,
|
||||||
|
it.lastPageRead,
|
||||||
|
it.fetchedAt.seconds.inWholeMilliseconds,
|
||||||
|
it.uploadDate,
|
||||||
|
it.chapterNumber,
|
||||||
|
chapters.size - it.index,
|
||||||
|
).apply {
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
this.meta = chapterToMeta[it.id] ?: emptyMap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (flags.includeHistory) {
|
||||||
|
backupManga.history =
|
||||||
|
chapters.mapNotNull {
|
||||||
|
if (it.lastReadAt > 0) {
|
||||||
|
BackupHistory(
|
||||||
|
url = it.url,
|
||||||
|
lastRead = it.lastReadAt.seconds.inWholeMilliseconds,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeCategories) {
|
||||||
|
backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeTracking) {
|
||||||
|
val tracks =
|
||||||
|
Tracker.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
|
||||||
|
if (it.record == null) {
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
BackupTracking(
|
||||||
|
syncId = it.record.trackerId,
|
||||||
|
// forced not null so its compatible with 1.x backup system
|
||||||
|
libraryId = it.record.libraryId ?: 0,
|
||||||
|
mediaId = it.record.remoteId,
|
||||||
|
title = it.record.title,
|
||||||
|
lastChapterRead = it.record.lastChapterRead.toFloat(),
|
||||||
|
totalChapters = it.record.totalChapters,
|
||||||
|
score = it.record.score.toFloat(),
|
||||||
|
status = it.record.status,
|
||||||
|
startedReadingDate = it.record.startDate,
|
||||||
|
finishedReadingDate = it.record.finishDate,
|
||||||
|
trackingUrl = it.record.remoteUrl,
|
||||||
|
private = it.record.private,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (tracks.isNotEmpty()) {
|
||||||
|
backupManga.tracking = tracks
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
backupManga
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun restore(
|
||||||
|
backupManga: BackupManga,
|
||||||
|
categoryMapping: Map<Int, Int>,
|
||||||
|
sourceMapping: Map<Long, String>,
|
||||||
|
errors: MutableList<Pair<Date, String>>,
|
||||||
|
flags: BackupFlags,
|
||||||
|
) {
|
||||||
|
val chapters = backupManga.chapters
|
||||||
|
val categories = backupManga.categories
|
||||||
|
val history = backupManga.history
|
||||||
|
val tracking = backupManga.tracking
|
||||||
|
|
||||||
|
val dbCategoryIds = categories.mapNotNull { categoryMapping[it] }
|
||||||
|
|
||||||
|
try {
|
||||||
|
restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags)
|
||||||
|
} catch (e: Exception) {
|
||||||
|
val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString()
|
||||||
|
errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun restoreMangaData(
|
||||||
|
manga: BackupManga,
|
||||||
|
chapters: List<BackupChapter>,
|
||||||
|
categoryIds: List<Int>,
|
||||||
|
history: List<BackupHistory>,
|
||||||
|
tracks: List<BackupTracking>,
|
||||||
|
flags: BackupFlags,
|
||||||
|
) {
|
||||||
|
val dbManga =
|
||||||
|
transaction {
|
||||||
|
MangaTable
|
||||||
|
.selectAll()
|
||||||
|
.where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
|
||||||
|
.firstOrNull()
|
||||||
|
}
|
||||||
|
val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW
|
||||||
|
|
||||||
|
val mangaId =
|
||||||
|
transaction {
|
||||||
|
val mangaId =
|
||||||
|
if (dbManga == null) {
|
||||||
|
// insert manga to database
|
||||||
|
MangaTable
|
||||||
|
.insertAndGetId {
|
||||||
|
it[url] = manga.url
|
||||||
|
it[title] = manga.title
|
||||||
|
|
||||||
|
it[artist] = manga.artist
|
||||||
|
it[author] = manga.author
|
||||||
|
it[description] = manga.description
|
||||||
|
it[genre] = manga.genre.joinToString()
|
||||||
|
it[status] = manga.status
|
||||||
|
it[thumbnail_url] = manga.thumbnailUrl
|
||||||
|
it[updateStrategy] = manga.updateStrategy.name
|
||||||
|
|
||||||
|
it[sourceReference] = manga.source
|
||||||
|
|
||||||
|
it[initialized] = manga.description != null
|
||||||
|
|
||||||
|
it[inLibrary] = manga.favorite
|
||||||
|
|
||||||
|
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
|
||||||
|
}.value
|
||||||
|
} else {
|
||||||
|
val dbMangaId = dbManga[MangaTable.id].value
|
||||||
|
|
||||||
|
// Merge manga data
|
||||||
|
MangaTable.update({ MangaTable.id eq dbMangaId }) {
|
||||||
|
it[artist] = manga.artist ?: dbManga[artist]
|
||||||
|
it[author] = manga.author ?: dbManga[author]
|
||||||
|
it[description] = manga.description ?: dbManga[description]
|
||||||
|
it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre]
|
||||||
|
it[status] = manga.status
|
||||||
|
it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url]
|
||||||
|
it[updateStrategy] = manga.updateStrategy.name
|
||||||
|
|
||||||
|
it[initialized] = dbManga[initialized] || manga.description != null
|
||||||
|
|
||||||
|
it[inLibrary] = manga.favorite || dbManga[inLibrary]
|
||||||
|
|
||||||
|
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
|
||||||
|
}
|
||||||
|
|
||||||
|
dbMangaId
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete thumbnail in case cached data still exists
|
||||||
|
clearThumbnail(mangaId)
|
||||||
|
|
||||||
|
if (flags.includeClientData && manga.meta.isNotEmpty()) {
|
||||||
|
modifyMangasMetas(mapOf(mangaId to manga.meta))
|
||||||
|
}
|
||||||
|
|
||||||
|
// merge chapter data
|
||||||
|
if (flags.includeChapters || flags.includeHistory) {
|
||||||
|
restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
// merge categories
|
||||||
|
if (flags.includeCategories) {
|
||||||
|
restoreMangaCategoryData(mangaId, categoryIds)
|
||||||
|
}
|
||||||
|
|
||||||
|
mangaId
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeTracking) {
|
||||||
|
restoreMangaTrackerData(mangaId, tracks)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: insert/merge history
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getMangaChapterToRestoreInfo(
|
||||||
|
mangaId: Int,
|
||||||
|
restoreMode: RestoreMode,
|
||||||
|
chapters: List<BackupChapter>,
|
||||||
|
): Pair<List<BackupChapter>, List<Pair<BackupChapter, ResultRow>>> {
|
||||||
|
val uniqueChapters = chapters.distinctBy { it.url }
|
||||||
|
|
||||||
|
if (restoreMode == RestoreMode.NEW) {
|
||||||
|
return Pair(uniqueChapters, emptyList())
|
||||||
|
}
|
||||||
|
|
||||||
|
val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] }
|
||||||
|
|
||||||
|
val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) }
|
||||||
|
val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! }
|
||||||
|
|
||||||
|
return chaptersToInsert to chaptersToUpdateToDbChapter
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun restoreMangaChapterData(
|
||||||
|
mangaId: Int,
|
||||||
|
restoreMode: RestoreMode,
|
||||||
|
chapters: List<BackupChapter>,
|
||||||
|
history: List<BackupHistory>,
|
||||||
|
flags: BackupFlags,
|
||||||
|
) = dbTransaction {
|
||||||
|
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
|
||||||
|
val historyByChapter = history.groupBy({ it.url }, { it.lastRead })
|
||||||
|
|
||||||
|
val insertedChapterIds =
|
||||||
|
if (flags.includeChapters) {
|
||||||
|
ChapterTable
|
||||||
|
.batchInsert(chaptersToInsert) { chapter ->
|
||||||
|
this[ChapterTable.url] = chapter.url
|
||||||
|
this[ChapterTable.name] = chapter.name
|
||||||
|
if (chapter.dateUpload == 0L) {
|
||||||
|
this[ChapterTable.date_upload] = chapter.dateFetch
|
||||||
|
} else {
|
||||||
|
this[ChapterTable.date_upload] = chapter.dateUpload
|
||||||
|
}
|
||||||
|
this[ChapterTable.chapter_number] = chapter.chapterNumber
|
||||||
|
this[ChapterTable.scanlator] = chapter.scanlator
|
||||||
|
|
||||||
|
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
|
||||||
|
this[ChapterTable.manga] = mangaId
|
||||||
|
|
||||||
|
this[ChapterTable.isRead] = chapter.read
|
||||||
|
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
|
||||||
|
this[ChapterTable.isBookmarked] = chapter.bookmark
|
||||||
|
|
||||||
|
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
|
||||||
|
|
||||||
|
if (flags.includeHistory) {
|
||||||
|
this[ChapterTable.lastReadAt] =
|
||||||
|
historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
|
||||||
|
}
|
||||||
|
}.map { it[ChapterTable.id].value }
|
||||||
|
} else {
|
||||||
|
emptyList()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
|
||||||
|
BatchUpdateStatement(ChapterTable).apply {
|
||||||
|
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
|
||||||
|
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
|
||||||
|
if (flags.includeChapters) {
|
||||||
|
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
|
||||||
|
this[ChapterTable.lastPageRead] =
|
||||||
|
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
|
||||||
|
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeHistory) {
|
||||||
|
this[ChapterTable.lastReadAt] =
|
||||||
|
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
|
||||||
|
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
execute(this@dbTransaction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
|
||||||
|
val chapterToUpdateByChapterId =
|
||||||
|
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
|
||||||
|
dbChapter[ChapterTable.id].value to
|
||||||
|
backupChapter
|
||||||
|
}
|
||||||
|
val metaEntryByChapterId =
|
||||||
|
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
|
||||||
|
.associate { (chapterId, backupChapter) ->
|
||||||
|
chapterId to backupChapter.meta
|
||||||
|
}
|
||||||
|
|
||||||
|
modifyChaptersMetas(metaEntryByChapterId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun restoreMangaCategoryData(
|
||||||
|
mangaId: Int,
|
||||||
|
categoryIds: List<Int>,
|
||||||
|
) {
|
||||||
|
CategoryManga.addMangaToCategories(mangaId, categoryIds)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun restoreMangaTrackerData(
|
||||||
|
mangaId: Int,
|
||||||
|
tracks: List<BackupTracking>,
|
||||||
|
) {
|
||||||
|
val dbTrackRecordsByTrackerId =
|
||||||
|
Tracker
|
||||||
|
.getTrackRecordsByMangaId(mangaId)
|
||||||
|
.mapNotNull { it.record?.toTrack() }
|
||||||
|
.associateBy { it.tracker_id }
|
||||||
|
|
||||||
|
val (existingTracks, newTracks) =
|
||||||
|
tracks
|
||||||
|
.mapNotNull { backupTrack ->
|
||||||
|
val track = backupTrack.toTrack(mangaId)
|
||||||
|
|
||||||
|
val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null
|
||||||
|
if (isUnsupportedTracker) {
|
||||||
|
return@mapNotNull null
|
||||||
|
}
|
||||||
|
|
||||||
|
val dbTrack =
|
||||||
|
dbTrackRecordsByTrackerId[backupTrack.syncId]
|
||||||
|
?: // new track
|
||||||
|
return@mapNotNull track
|
||||||
|
|
||||||
|
if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
|
||||||
|
return@mapNotNull null
|
||||||
|
}
|
||||||
|
|
||||||
|
dbTrack.also {
|
||||||
|
it.remote_id = track.remote_id
|
||||||
|
it.library_id = track.library_id
|
||||||
|
it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
|
||||||
|
}
|
||||||
|
}.partition { (it.id ?: -1) > 0 }
|
||||||
|
|
||||||
|
Tracker.updateTrackRecords(existingTracks)
|
||||||
|
Tracker.insertTrackRecords(newTracks)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright (C) Contributors to the Suwayomi project
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||||
|
|
||||||
|
import org.jetbrains.exposed.sql.selectAll
|
||||||
|
import suwayomi.tachidesk.manga.impl.Source
|
||||||
|
import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
|
||||||
|
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
|
||||||
|
import suwayomi.tachidesk.manga.model.table.SourceTable
|
||||||
|
import suwayomi.tachidesk.server.database.dbTransaction
|
||||||
|
|
||||||
|
object BackupSourceHandler {
|
||||||
|
fun backup(
|
||||||
|
backupMangas: List<BackupManga>,
|
||||||
|
flags: BackupFlags,
|
||||||
|
): List<BackupSource> =
|
||||||
|
dbTransaction {
|
||||||
|
val inLibraryMangaSourceIds =
|
||||||
|
backupMangas
|
||||||
|
.asSequence()
|
||||||
|
.map { it.source }
|
||||||
|
.distinct()
|
||||||
|
.toList()
|
||||||
|
val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds }
|
||||||
|
val sourceToMeta =
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value })
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
inLibraryMangaSourceIds
|
||||||
|
.map { mangaSourceId ->
|
||||||
|
val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId }
|
||||||
|
BackupSource(
|
||||||
|
source?.get(SourceTable.name) ?: "",
|
||||||
|
mangaSourceId,
|
||||||
|
).apply {
|
||||||
|
if (flags.includeClientData) {
|
||||||
|
this.meta = sourceToMeta[mangaSourceId] ?: emptyMap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun restore(backupSources: List<BackupSource>) {
|
||||||
|
modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta })
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user