Fix/inserting duplicated chapters into database (#991)

* Prevent adding duplicated chapters into the db

it's possible that the source returns a list containing chapters with the same url
once such duplicated chapters have been added, they aren't being removed anymore as long as there is
a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself
does not exist anymore on the source

* Drop duplicated chapters from database table

* Add unique constraint to chapter table

This is to completely prevent duplicated chapters from being added to the database.
Since once a duplicated chapter has been added to the database, it does not get removed anymore as long as a chapter with the same url is included in the requested source chapter list

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -145,14 +145,20 @@ object Chapter {
 
                 val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f
                 val numberOfCurrentChapters = getCountOfMangaChapters(mangaId)
-                val chapterList = source.getChapterList(sManga)
 
-                if (chapterList.isEmpty()) {
+                val chapters = source.getChapterList(sManga)
+                // it's possible that the source returns a list containing chapters with the same url
+                // once such duplicated chapters have been added, they aren't being removed anymore as long as there is
+                // a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself
+                // does not exist anymore on the source
+                val uniqueChapters = chapters.distinctBy { it.url }
+
+                if (uniqueChapters.isEmpty()) {
                     throw Exception("No chapters found")
                 }
 
                 // Recognize number for new chapters.
-                chapterList.forEach { chapter ->
+                uniqueChapters.forEach { chapter ->
                     (source as? HttpSource)?.prepareNewChapter(chapter, sManga)
                     val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble())
                     chapter.chapter_number = chapterNumber.toFloat()
@@ -178,7 +184,7 @@ object Chapter {
                 val chaptersToInsert = mutableListOf<ChapterDataClass>() // do not yet have an ID from the database
                 val chaptersToUpdate = mutableListOf<ChapterDataClass>()
 
-                chapterList.reversed().forEachIndexed { index, fetchedChapter ->
+                uniqueChapters.reversed().forEachIndexed { index, fetchedChapter ->
                     val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url }
 
                     val chapterData =
@@ -221,7 +227,7 @@ object Chapter {
                 val deletedChapterNumberDateFetchMap = mutableMapOf<Float, Long>()
 
                 // clear any orphaned/duplicate chapters that are in the db but not in `chapterList`
-                val chapterUrls = chapterList.map { it.url }.toSet()
+                val chapterUrls = uniqueChapters.map { it.url }.toSet()
 
                 val chaptersIdsToDelete =
                     chaptersInDb.mapNotNull { dbChapter ->
@@ -298,7 +304,7 @@ object Chapter {
                     downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters)
                 }
 
-                chapterList
+                uniqueChapters
             }
 
         return chapterList

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0039_DeleteDuplicatedChapters.kt

@@ -0,0 +1,23 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.SQLMigration
+
+@Suppress("ClassName", "unused")
+class M0039_DeleteDuplicatedChapters : SQLMigration() {
+    override val sql: String =
+        """
+        DELETE FROM CHAPTER
+        WHERE id NOT IN (
+            SELECT MIN(id)
+            FROM CHAPTER
+            GROUP BY URL, MANGA
+        );
+        """.trimIndent()
+}

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0040_AddUniqueConstraintToChapterTable.kt

@@ -0,0 +1,19 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.SQLMigration
+
+@Suppress("ClassName", "unused")
+class M0040_AddUniqueConstraintToChapterTable : SQLMigration() {
+    override val sql: String =
+        """
+        ALTER TABLE CHAPTER
+            ADD CONSTRAINT UC_CHAPTER UNIQUE (URL, MANGA)
+        """.trimIndent()
+}