Feature/backup tracking (#940)

* Include tracking in validation of backup

* Always return track records

Not clear why an empty list should be returned in case no trackers are logged in

* Include tracking in backup creation

* Restore tracking from backup

server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/BackupQuery.kt

@@ -16,14 +16,20 @@ class BackupQuery {
         val name: String,
     )
 
+    data class ValidateBackupTracker(
+        val name: String,
+    )
+
     data class ValidateBackupResult(
         val missingSources: List<ValidateBackupSource>,
+        val missingTrackers: List<ValidateBackupTracker>,
     )
 
     fun validateBackup(input: ValidateBackupInput): ValidateBackupResult {
         val result = ProtoBackupValidator.validate(input.backup.content)
         return ValidateBackupResult(
             result.missingSourceIds.map { ValidateBackupSource(it.first, it.second) },
+            result.missingTrackers.map { ValidateBackupTracker(it) },
         )
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/Track.kt

@@ -11,7 +11,7 @@ interface Track : Serializable {
 
     var sync_id: Int
 
-    var media_id: Int
+    var media_id: Long
 
     var library_id: Long?
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/TrackImpl.kt

@@ -9,7 +9,7 @@ class TrackImpl : Track {
 
     override var sync_id: Int = 0
 
-    override var media_id: Int = 0
+    override var media_id: Long = 0L
 
     override var library_id: Long? = null
 
@@ -43,7 +43,7 @@ class TrackImpl : Track {
     override fun hashCode(): Int {
         var result = (manga_id xor manga_id.ushr(32)).toInt()
         result = 31 * result + sync_id
-        result = 31 * result + media_id
+        result = (31 * result + media_id).toInt()
         return result
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

@@ -31,6 +31,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
+import suwayomi.tachidesk.manga.impl.track.Track
 import suwayomi.tachidesk.manga.model.table.CategoryTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaStatus
@@ -230,9 +232,32 @@ object ProtoBackupExport : ProtoBackupBase() {
                 backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
             }
 
-//            if(flags.includeTracking) {
+            if (flags.includeTracking) {
-//                backupManga.tracking = TODO()
+                val tracks =
-//            }
+                    Track.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
+                        if (it.record == null) {
+                            null
+                        } else {
+                            BackupTracking(
+                                syncId = it.record.trackerId,
+                                // forced not null so its compatible with 1.x backup system
+                                libraryId = it.record.libraryId ?: 0,
+                                mediaId = it.record.remoteId,
+                                title = it.record.title,
+                                lastChapterRead = it.record.lastChapterRead.toFloat(),
+                                totalChapters = it.record.totalChapters,
+                                score = it.record.score.toFloat(),
+                                status = it.record.status,
+                                startedReadingDate = it.record.startDate,
+                                finishedReadingDate = it.record.finishDate,
+                                trackingUrl = it.record.remoteUrl,
+                            )
+                        }
+                    }
+                if (tracks.isNotEmpty()) {
+                    backupManga.tracking = tracks
+                }
+            }
 
 //            if (flags.includeHistory) {
 //                backupManga.history = TODO()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -34,22 +34,26 @@ import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
 import suwayomi.tachidesk.manga.impl.backup.models.Chapter
 import suwayomi.tachidesk.manga.impl.backup.models.Manga
-import suwayomi.tachidesk.manga.impl.backup.models.Track
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
+import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
+import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
+import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
 import suwayomi.tachidesk.manga.model.table.CategoryTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import java.io.InputStream
-import java.lang.Integer.max
 import java.util.Date
 import java.util.Timer
 import java.util.TimerTask
 import java.util.concurrent.TimeUnit
+import kotlin.math.max
+import suwayomi.tachidesk.manga.impl.track.Track as Tracker
 
 object ProtoBackupImport : ProtoBackupBase() {
     private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@@ -239,10 +243,9 @@ object ProtoBackupImport : ProtoBackupBase() {
         val chapters = backupManga.getChaptersImpl()
         val categories = backupManga.categories
         val history = backupManga.brokenHistory.map { BackupHistory(it.url, it.lastRead) } + backupManga.history
-        val tracks = backupManga.getTrackingImpl()
 
         try {
-            restoreMangaData(manga, chapters, categories, history, tracks, backupCategories, categoryMapping)
+            restoreMangaData(manga, chapters, categories, history, backupManga.tracking, backupCategories, categoryMapping)
         } catch (e: Exception) {
             val sourceName = sourceMapping[manga.source] ?: manga.source.toString()
             errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}")
@@ -255,7 +258,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         chapters: List<Chapter>,
         categories: List<Int>,
         history: List<BackupHistory>,
-        tracks: List<Track>,
+        tracks: List<BackupTracking>,
         backupCategories: List<BackupCategory>,
         categoryMapping: Map<Int, Int>,
     ) {
@@ -265,6 +268,7 @@ object ProtoBackupImport : ProtoBackupBase() {
                     .firstOrNull()
             }
 
+        val mangaId =
             if (dbManga == null) { // Manga not in database
                 transaction {
                     // insert manga to database
@@ -320,6 +324,8 @@ object ProtoBackupImport : ProtoBackupBase() {
                     categories.forEach { backupCategoryOrder ->
                         CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
                     }
+
+                    mangaId
                 }
             } else { // Manga in database
                 transaction {
@@ -381,11 +387,40 @@ object ProtoBackupImport : ProtoBackupBase() {
                     categories.forEach { backupCategoryOrder ->
                         CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
                     }
+
+                    mangaId
                 }
             }
 
+        val dbTrackRecordsByTrackerId =
+            Tracker.getTrackRecordsByMangaId(mangaId)
+                .mapNotNull { it.record?.toTrack() }
+                .associateBy { it.sync_id }
+
+        val (existingTracks, newTracks) =
+            tracks.mapNotNull { backupTrack ->
+                val track = backupTrack.toTrack(mangaId)
+                val dbTrack =
+                    dbTrackRecordsByTrackerId[backupTrack.syncId]
+                        ?: // new track
+                        return@mapNotNull track
+
+                if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
+                    return@mapNotNull null
+                }
+
+                dbTrack.also {
+                    it.media_id = track.media_id
+                    it.library_id = track.library_id
+                    it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
+                }
+            }.partition { (it.id ?: -1) > 0 }
+
+        existingTracks.forEach(Tracker::updateTrackRecord)
+        newTracks.forEach(Tracker::insertTrackRecord)
+
         // TODO: insert/merge history
+    }
 
-        // TODO: insert/merge tracking
+    private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt

@@ -15,6 +15,7 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
 import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
+import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
 import suwayomi.tachidesk.manga.model.table.SourceTable
 import java.io.InputStream
 
@@ -39,17 +40,18 @@ object ProtoBackupValidator {
                 sources.filter { SourceTable.select { SourceTable.id eq it.key }.firstOrNull() == null }
             }
 
-//        val trackers = backup.backupManga
+        val trackers =
-//            .flatMap { it.tracking }
+            backup.backupManga
-//            .map { it.syncId }
+                .flatMap { it.tracking }
-//            .distinct()
+                .map { it.syncId }
+                .distinct()
 
-        val missingTrackers = listOf("")
+        val missingTrackers =
-//        val missingTrackers = trackers
+            trackers
-//            .mapNotNull { trackManager.getService(it) }
+                .mapNotNull { TrackerManager.getTracker(it) }
-//            .filter { !it.isLogged }
+                .filter { !it.isLoggedIn }
-//            .map { context.getString(it.nameRes()) }
+                .map { it.name }
-//            .sorted()
+                .sorted()
 
         return ValidationResult(
             missingSources

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupTracking.kt

@@ -8,11 +8,12 @@ import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
 @Serializable
 data class BackupTracking(
     // in 1.x some of these values have different types or names
-    // syncId is called siteId in 1,x
     @ProtoNumber(1) var syncId: Int,
     // LibraryId is not null in 1.x
     @ProtoNumber(2) var libraryId: Long,
-    @ProtoNumber(3) var mediaId: Int = 0,
+    @Deprecated("Use mediaId instead", level = DeprecationLevel.WARNING)
+    @ProtoNumber(3)
+    var mediaIdInt: Int = 0,
     // trackingUrl is called mediaUrl in 1.x
     @ProtoNumber(4) var trackingUrl: String = "",
     @ProtoNumber(5) var title: String = "",
@@ -25,11 +26,17 @@ data class BackupTracking(
     @ProtoNumber(10) var startedReadingDate: Long = 0,
     // finishedReadingDate is called endReadTime in 1.x
     @ProtoNumber(11) var finishedReadingDate: Long = 0,
+    @ProtoNumber(100) var mediaId: Long = 0,
 ) {
     fun getTrackingImpl(): TrackImpl {
         return TrackImpl().apply {
             sync_id = this@BackupTracking.syncId
-            media_id = this@BackupTracking.mediaId
+            media_id =
+                if (this@BackupTracking.mediaIdInt != 0) {
+                    this@BackupTracking.mediaIdInt.toLong()
+                } else {
+                    this@BackupTracking.mediaId
+                }
             library_id = this@BackupTracking.libraryId
             title = this@BackupTracking.title
             // convert from float to int because of 1.x types

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/track/Track.kt

@@ -74,9 +74,6 @@ object Track {
     }
 
     fun getTrackRecordsByMangaId(mangaId: Int): List<MangaTrackerDataClass> {
-        if (!TrackerManager.hasLoggedTracker()) {
-            return emptyList()
-        }
         val recordMap =
             transaction {
                 TrackRecordTable.select { TrackRecordTable.mangaId eq mangaId }
@@ -342,7 +339,7 @@ object Track {
         }
     }
 
-    private fun upsertTrackRecord(track: Track): Int {
+    fun upsertTrackRecord(track: Track): Int {
         return transaction {
             val existingRecord =
                 TrackRecordTable.select {
@@ -352,10 +349,22 @@ object Track {
                     .singleOrNull()
 
             if (existingRecord != null) {
-                TrackRecordTable.update({
+                updateTrackRecord(track)
+                existingRecord[TrackRecordTable.id].value
+            } else {
+                insertTrackRecord(track)
+            }
+        }
+    }
+
+    fun updateTrackRecord(track: Track): Int =
+        transaction {
+            TrackRecordTable.update(
+                {
                     (TrackRecordTable.mangaId eq track.manga_id) and
                         (TrackRecordTable.trackerId eq track.sync_id)
-                }) {
+                },
+            ) {
                 it[remoteId] = track.media_id
                 it[libraryId] = track.library_id
                 it[title] = track.title
@@ -367,8 +376,10 @@ object Track {
                 it[startDate] = track.started_reading_date
                 it[finishDate] = track.finished_reading_date
             }
-                existingRecord[TrackRecordTable.id].value
+        }
-            } else {
+
+    fun insertTrackRecord(track: Track): Int =
+        transaction {
             TrackRecordTable.insertAndGetId {
                 it[mangaId] = track.manga_id
                 it[trackerId] = track.sync_id
@@ -384,8 +395,6 @@ object Track {
                 it[finishDate] = track.finished_reading_date
             }.value
         }
-        }
-    }
 
     @Serializable
     data class LoginInput(

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/track/tracker/model/TrackConvertor.kt

@@ -1,8 +1,11 @@
 package suwayomi.tachidesk.manga.impl.track.tracker.model
 
 import org.jetbrains.exposed.sql.ResultRow
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
 import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
 import suwayomi.tachidesk.manga.model.table.TrackRecordTable
+import suwayomi.tachidesk.manga.model.table.TrackRecordTable.lastChapterRead
+import suwayomi.tachidesk.manga.model.table.TrackRecordTable.remoteUrl
 
 fun ResultRow.toTrackRecordDataClass(): TrackRecordDataClass =
     TrackRecordDataClass(
@@ -36,3 +39,52 @@ fun ResultRow.toTrack(): Track =
         it.started_reading_date = this[TrackRecordTable.startDate]
         it.finished_reading_date = this[TrackRecordTable.finishDate]
     }
+
+fun BackupTracking.toTrack(mangaId: Int): Track =
+    Track.create(syncId).also {
+        it.id = -1
+        it.manga_id = mangaId
+        it.media_id = mediaId
+        it.library_id = libraryId
+        it.title = title
+        it.last_chapter_read = lastChapterRead
+        it.total_chapters = totalChapters
+        it.status = status
+        it.score = score
+        it.tracking_url = trackingUrl
+        it.started_reading_date = startedReadingDate
+        it.finished_reading_date = finishedReadingDate
+    }
+
+fun TrackRecordDataClass.toTrack(): Track =
+    Track.create(trackerId).also {
+        it.id = id
+        it.manga_id = mangaId
+        it.media_id = remoteId
+        it.library_id = libraryId
+        it.title = title
+        it.last_chapter_read = lastChapterRead.toFloat()
+        it.total_chapters = totalChapters
+        it.status = status
+        it.score = score.toFloat()
+        it.tracking_url = remoteUrl
+        it.started_reading_date = startDate
+        it.finished_reading_date = finishDate
+    }
+
+fun Track.toTrackRecordDataClass(): TrackRecordDataClass =
+    TrackRecordDataClass(
+        id = id ?: -1,
+        mangaId = manga_id,
+        trackerId = sync_id,
+        remoteId = media_id,
+        libraryId = library_id,
+        title = title,
+        lastChapterRead = last_chapter_read.toDouble(),
+        totalChapters = total_chapters,
+        status = status,
+        score = score.toDouble(),
+        remoteUrl = tracking_url,
+        startDate = started_reading_date,
+        finishDate = finished_reading_date,
+    )