trying to bundle the changes

Makes it possible to only update specific categories.

In case a manga is in an excluded category it will be excluded even if it is also in an included category.

.github/workflows/issue_moderator.yml

@@ -1,24 +1,35 @@
-name: Issue closer
+name: Issue moderator
 
 on:
   issues:
     types: [opened, edited, reopened]
+  issue_comment:
+    types: [created]
 
 jobs:
   autoclose:
     runs-on: ubuntu-latest
     steps:
-      - name: Autoclose issues
+      - name: Moderate issues
-        uses: arkon/issue-closer-action@v3.0
+        uses: tachiyomiorg/issue-moderator-action@v1
         with:
           repo-token: ${{ github.token }}
-          rules: |
+          duplicate-check-enabled: true
+          duplicate-check-label: Source request
+          existing-check-enabled: true
+          existing-check-label: Source request
+          auto-close-rules: |
             [
               {
                 "type": "title",
-                "regex": ".*<short description>*",
+                "regex": ".*<short description>.*",
                 "message": "You did not fill out the description in the title"
               },
+              {
+                "type": "title",
+                "regex": ".*(<|>)+.*",
+                "message": "You did not remove Angle brackets(< and >) from the title"
+              },
               {
                 "type": "body",
                 "regex": ".*DELETE THIS SECTION IF YOU HAVE READ AND ACKNOWLEDGED IT.*",
@@ -26,7 +37,7 @@ jobs:
               },
               {
                 "type": "body",
-                "regex": "(Tachidesk version|Server Operating System|Server Desktop Environment|Server JVM version|Client Operating System|Client Web Browser):.*(\\(Example:|<usually).*",
+                "regex": ".*(Tachidesk version|Server Operating System|Server Desktop Environment|Server JVM version|Client Operating System|Client Web Browser):.*(\\(Example:|<usually).*",
                 "message": "The requested information was not filled out"
               },
               {

.github/workflows/winget.yml

@@ -1,12 +1,14 @@
 name: Publish to WinGet
 on:
-  release:
+  workflow_run:
-    types: [released]
+    workflows: ["CI Publish"]
+    types:
+      - completed
 jobs:
   publish:
     runs-on: windows-latest # action can only be run on windows
     steps:
-      - uses: vedantmgoyal2009/winget-releaser@v1
+      - uses: vedantmgoyal2009/winget-releaser@v2
         with:
           identifier: Suwayomi.Tachidesk-Server
           installers-regex: '.*x64.msi$'

.gitignore

@@ -2,7 +2,7 @@
 .gradle
 .idea
 gradle.properties
-
+.fleet
 # But we need these
 !.idea/runConfigurations
 

AndroidCompat/Config/build.gradle.kts

@@ -0,0 +1,12 @@
+@Suppress("DSL_SCOPE_VIOLATION")
+plugins {
+    id(libs.plugins.kotlin.jvm.get().pluginId)
+    id(libs.plugins.kotlin.serialization.get().pluginId)
+    id(libs.plugins.kotlinter.get().pluginId)
+}
+
+dependencies {
+    // Shared
+    implementation(libs.bundles.shared)
+    testImplementation(libs.bundles.sharedTest)
+}

AndroidCompat/build.gradle.kts

@@ -1,28 +1,39 @@
+@Suppress("DSL_SCOPE_VIOLATION")
+plugins {
+    id(libs.plugins.kotlin.jvm.get().pluginId)
+    id(libs.plugins.kotlin.serialization.get().pluginId)
+    id(libs.plugins.kotlinter.get().pluginId)
+}
+
 dependencies {
+    // Shared
+    implementation(libs.bundles.shared)
+    testImplementation(libs.bundles.sharedTest)
+
     // Android stub library
-    implementation("com.github.Suwayomi:android-jar:1.0.0")
+    implementation(libs.android.stubs)
 
     // XML
-    compileOnly("xmlpull:xmlpull:1.1.3.4a")
+    compileOnly(libs.xmlpull)
 
     // Config API
-    implementation(project(":AndroidCompat:Config"))
+    implementation(projects.androidCompat.config)
 
     // APK sig verifier
-    compileOnly("com.android.tools.build:apksig:7.2.1")
+    compileOnly(libs.apksig)
 
     // AndroidX annotations
-    compileOnly("androidx.annotation:annotation:1.5.0")
+    compileOnly(libs.android.annotations)
 
     // substitute for duktape-android
-    implementation("org.mozilla:rhino-runtime:1.7.14") // slimmer version of 'org.mozilla:rhino'
+    implementation(libs.bundles.rhino)
-    implementation("org.mozilla:rhino-engine:1.7.14") // provides the same interface as 'javax.script' a.k.a Nashorn
 
     // Kotlin wrapper around Java Preferences, makes certain things easier
-    val multiplatformSettingsVersion = "1.0.0-RC"
+    implementation(libs.bundles.settings)
-    implementation("com.russhwolf:multiplatform-settings-jvm:$multiplatformSettingsVersion")
-    implementation("com.russhwolf:multiplatform-settings-serialization-jvm:$multiplatformSettingsVersion")
 
     // Android version of SimpleDateFormat
-    implementation("com.ibm.icu:icu4j:72.1")
+    implementation(libs.icu4j)
+
+    // OpenJDK lacks native JPEG encoder and native WEBP decoder
+    implementation(libs.bundles.twelvemonkeys)
 }

AndroidCompat/src/main/java/android/graphics/Bitmap.java

@@ -0,0 +1,129 @@
+package android.graphics;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
+import javax.imageio.IIOImage;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageWriteParam;
+import javax.imageio.ImageWriter;
+import javax.imageio.stream.ImageOutputStream;
+
+public final class Bitmap {
+    private int width;
+    private int height;
+    private BufferedImage image;
+
+    public Bitmap(BufferedImage image) {
+        this.image = image;
+        this.width = image.getWidth();
+        this.height = image.getHeight();
+    }
+
+    public BufferedImage getImage() {
+        return image;
+    }
+
+    public int getHeight() {
+        return height;
+    }
+
+    public int getWidth() {
+        return width;
+    }
+
+    public enum CompressFormat {
+        JPEG          (0),
+        PNG           (1),
+        WEBP          (2),
+        WEBP_LOSSY    (3),
+        WEBP_LOSSLESS (4);
+
+        CompressFormat(int nativeInt) {
+            this.nativeInt = nativeInt;
+        }
+
+        final int nativeInt;
+    }
+
+    public enum Config {
+        ALPHA_8(1),
+        RGB_565(3),
+        ARGB_4444(4),
+        ARGB_8888(5),
+        RGBA_F16(6),
+        HARDWARE(7),
+        RGBA_1010102(8);
+
+        final int nativeInt;
+
+        private static Config sConfigs[] = {
+            null, ALPHA_8, null, RGB_565, ARGB_4444, ARGB_8888, RGBA_F16, HARDWARE, RGBA_1010102
+        };
+
+        Config(int ni) {
+            this.nativeInt = ni;
+        }
+
+        static Config nativeToConfig(int ni) {
+            return sConfigs[ni];
+        }
+    }
+
+    public static Bitmap createBitmap(int width, int height, Config config) {
+        BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+        return new Bitmap(image);
+    }
+
+    public boolean compress(CompressFormat format, int quality, OutputStream stream) {
+        if (stream == null) {
+            throw new NullPointerException();
+        }
+
+        if (quality < 0 || quality > 100) {
+            throw new IllegalArgumentException("quality must be 0..100");
+        }
+        float qualityFloat = ((float) quality) / 100;
+
+        String formatString = "";
+        if (format == Bitmap.CompressFormat.PNG) {
+            formatString = "png";
+        } else if (format == Bitmap.CompressFormat.JPEG) {
+            formatString = "jpg";
+        } else {
+            throw new IllegalArgumentException("unsupported compression format!");
+        }
+
+        Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName(formatString);
+        if (!writers.hasNext()) {
+            throw new IllegalStateException("no image writers found for this format!");
+        }
+        ImageWriter writer = (ImageWriter) writers.next();
+
+        ImageOutputStream ios;
+        try {
+            ios = ImageIO.createImageOutputStream(stream);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        writer.setOutput(ios);
+
+        ImageWriteParam param = writer.getDefaultWriteParam();
+        if (formatString == "jpg") {
+            param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
+            param.setCompressionQuality(qualityFloat);
+        }
+
+        try {
+             writer.write(null, new IIOImage(image, null, null), param);
+             ios.close();
+             writer.dispose();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return true;
+    }
+}

AndroidCompat/src/main/java/android/graphics/BitmapFactory.java

@@ -0,0 +1,51 @@
+package android.graphics;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+
+public class BitmapFactory {
+    public static Bitmap decodeStream(InputStream inputStream) {
+        Bitmap bitmap = null;
+
+        try {
+            ImageInputStream imageInputStream = ImageIO.createImageInputStream(inputStream);
+            Iterator<ImageReader> imageReaders = ImageIO.getImageReaders(imageInputStream);
+
+            if (!imageReaders.hasNext()) {
+                throw new IllegalArgumentException("no reader for image");
+            }
+
+            ImageReader imageReader = imageReaders.next();
+            imageReader.setInput(imageInputStream);
+
+            BufferedImage image = imageReader.read(0, imageReader.getDefaultReadParam());
+            bitmap = new Bitmap(image);
+
+            imageReader.dispose();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return bitmap;
+    }
+
+    public static Bitmap decodeByteArray(byte[] data, int offset, int length) {
+        Bitmap bitmap = null;
+
+        ByteArrayInputStream byteArrayStream = new ByteArrayInputStream(data);
+        try {
+            BufferedImage image = ImageIO.read(byteArrayStream);
+            bitmap = new Bitmap(image);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return bitmap;
+    }
+}

AndroidCompat/src/main/java/android/graphics/Canvas.java

@@ -0,0 +1,21 @@
+package android.graphics;
+
+import java.awt.Graphics2D;
+import java.awt.image.BufferedImage;
+import javax.imageio.ImageIO;
+
+public final class Canvas {
+    private BufferedImage canvasImage;
+    private Graphics2D canvas;
+
+    public Canvas(Bitmap bitmap) {
+        canvasImage = bitmap.getImage();
+        canvas = canvasImage.createGraphics();
+    }
+
+    public void drawBitmap(Bitmap sourceBitmap, Rect src, Rect dst, Paint paint) {        
+        BufferedImage sourceImage = sourceBitmap.getImage();
+        BufferedImage sourceImageCropped = sourceImage.getSubimage(src.left, src.top, src.getWidth(), src.getHeight());
+        canvas.drawImage(sourceImageCropped, null, dst.left, dst.top);
+    }
+}

AndroidCompat/src/main/java/android/graphics/Rect.java

@@ -0,0 +1,122 @@
+package android.graphics;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public final class Rect {
+    int left;
+    int top;
+    int right;
+    int bottom;
+
+    private static final class UnflattenHelper {
+        private static final Pattern FLATTENED_PATTERN = Pattern.compile(
+            "(-?\\d+) (-?\\d+) (-?\\d+) (-?\\d+)");
+
+        static Matcher getMatcher(String str) {
+            return FLATTENED_PATTERN.matcher(str);
+        }
+    }
+
+    public Rect() {
+    }
+
+    public Rect(int left, int top, int right, int bottom) {
+        this.left = left;
+        this.top = top;
+        this.right = right;
+        this.bottom = bottom;
+    }
+
+    public Rect(Rect r) {
+        if (r == null) {
+            this.left = 0;
+            this.top = 0;
+            this.right = 0;
+            this.bottom = 0;
+        } else {
+            this.left = left;
+            this.top = top;
+            this.right = right;
+            this.bottom = bottom;
+        }
+    }
+
+    public final int getWidth() {
+        return right - left;
+    }
+
+    public final int getHeight() {
+        return bottom - top;
+    }
+
+    public static Rect unflattenFromString(String str) {
+        if (str.isEmpty()) {
+            return null;
+        }
+
+        Matcher matcher = UnflattenHelper.getMatcher(str);
+        if (!matcher.matches()) {
+            return null;
+        }
+
+        return new Rect(Integer.parseInt(matcher.group(1)),
+                        Integer.parseInt(matcher.group(2)),
+                        Integer.parseInt(matcher.group(3)),
+                        Integer.parseInt(matcher.group(4)));
+    }
+
+    public String toShortString() {
+        return toShortString(new StringBuilder(32));
+    }
+    
+    public String toShortString(StringBuilder sb) {
+        sb.setLength(0);
+        sb.append('['); sb.append(left); sb.append(',');
+        sb.append(top); sb.append("]["); sb.append(right);
+        sb.append(','); sb.append(bottom); sb.append(']');
+        return sb.toString();
+    }
+
+    public String flattenToString() {
+        StringBuilder sb = new StringBuilder(32);
+        sb.append(left);
+        sb.append(' ');
+        sb.append(top);
+        sb.append(' ');
+        sb.append(right);
+        sb.append(' ');
+        sb.append(bottom);
+        return sb.toString();
+    }
+
+    public void writeToParcel(Parcel out, int flags) {
+        out.writeInt(left);
+        out.writeInt(top);
+        out.writeInt(right);
+        out.writeInt(bottom);
+    }
+
+    public static final Parcelable.Creator<Rect> CREATOR = new Parcelable.Creator<Rect>() {
+        @Override
+        public Rect createFromParcel(Parcel in) {
+            Rect r = new Rect();
+            r.readFromParcel(in);
+            return r;
+        }
+
+        @Override
+        public Rect[] newArray(int size) {
+            return new Rect[size];
+        }
+    };
+
+    public void readFromParcel(Parcel in) {
+        left = in.readInt();
+        top = in.readInt();
+        right = in.readInt();
+        bottom = in.readInt();
+    }
+}

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/io/sharedprefs/JavaSharedPreferences.kt

@@ -9,7 +9,6 @@ package xyz.nulldev.androidcompat.io.sharedprefs
 
 import android.content.SharedPreferences
 import com.russhwolf.settings.ExperimentalSettingsApi
-import com.russhwolf.settings.ExperimentalSettingsImplementation
 import com.russhwolf.settings.PreferencesSettings
 import com.russhwolf.settings.serialization.decodeValue
 import com.russhwolf.settings.serialization.decodeValueOrNull
@@ -21,7 +20,7 @@ import kotlinx.serialization.builtins.serializer
 import java.util.prefs.PreferenceChangeListener
 import java.util.prefs.Preferences
 
-@OptIn(ExperimentalSettingsImplementation::class, ExperimentalSerializationApi::class, ExperimentalSettingsApi::class)
+@OptIn(ExperimentalSerializationApi::class, ExperimentalSettingsApi::class)
 class JavaSharedPreferences(key: String) : SharedPreferences {
     private val javaPreferences = Preferences.userRoot().node("suwayomi/tachidesk/$key")
     private val preferences = PreferencesSettings(javaPreferences)
@@ -77,13 +76,19 @@ class JavaSharedPreferences(key: String) : SharedPreferences {
     }
 
     class Editor(private val preferences: PreferencesSettings) : SharedPreferences.Editor {
-        val itemsToAdd = mutableMapOf<String, Any>()
+        private val actions = mutableListOf<Action>()
+
+        private sealed class Action {
+            data class Add(val key: String, val value: Any) : Action()
+            data class Remove(val key: String) : Action()
+            object Clear : Action()
+        }
 
         override fun putString(key: String, value: String?): SharedPreferences.Editor {
             if (value != null) {
-                itemsToAdd[key] = value
+                actions += Action.Add(key, value)
             } else {
-                remove(key)
+                actions += Action.Remove(key)
             }
             return this
         }
@@ -93,40 +98,40 @@ class JavaSharedPreferences(key: String) : SharedPreferences {
             values: MutableSet<String>?
         ): SharedPreferences.Editor {
             if (values != null) {
-                itemsToAdd[key] = values
+                actions += Action.Add(key, values)
             } else {
-                remove(key)
+                actions += Action.Remove(key)
             }
             return this
         }
 
         override fun putInt(key: String, value: Int): SharedPreferences.Editor {
-            itemsToAdd[key] = value
+            actions += Action.Add(key, value)
             return this
         }
 
         override fun putLong(key: String, value: Long): SharedPreferences.Editor {
-            itemsToAdd[key] = value
+            actions += Action.Add(key, value)
             return this
         }
 
         override fun putFloat(key: String, value: Float): SharedPreferences.Editor {
-            itemsToAdd[key] = value
+            actions += Action.Add(key, value)
             return this
         }
 
         override fun putBoolean(key: String, value: Boolean): SharedPreferences.Editor {
-            itemsToAdd[key] = value
+            actions += Action.Add(key, value)
             return this
         }
 
         override fun remove(key: String): SharedPreferences.Editor {
-            itemsToAdd.remove(key)
+            actions += Action.Remove(key)
             return this
         }
 
         override fun clear(): SharedPreferences.Editor {
-            itemsToAdd.clear()
+            actions.add(Action.Clear)
             return this
         }
 
@@ -140,16 +145,33 @@ class JavaSharedPreferences(key: String) : SharedPreferences {
         }
 
         private fun addToPreferences() {
-            itemsToAdd.forEach { (key, value) ->
+            actions.forEach {
                 @Suppress("UNCHECKED_CAST")
-                when (value) {
+                when (it) {
-                    is Set<*> -> preferences.encodeValue(SetSerializer(String.serializer()), key, value as Set<String>)
+                    is Action.Add -> when (val value = it.value) {
-                    is String -> preferences.putString(key, value)
+                        is Set<*> -> preferences.encodeValue(SetSerializer(String.serializer()), it.key, value as Set<String>)
-                    is Int -> preferences.putInt(key, value)
+                        is String -> preferences.putString(it.key, value)
-                    is Long -> preferences.putLong(key, value)
+                        is Int -> preferences.putInt(it.key, value)
-                    is Float -> preferences.putFloat(key, value)
+                        is Long -> preferences.putLong(it.key, value)
-                    is Double -> preferences.putDouble(key, value)
+                        is Float -> preferences.putFloat(it.key, value)
-                    is Boolean -> preferences.putBoolean(key, value)
+                        is Double -> preferences.putDouble(it.key, value)
+                        is Boolean -> preferences.putBoolean(it.key, value)
+                    }
+                    is Action.Remove -> {
+                        preferences.remove(it.key)
+                        /**
+                         * Set<String> are stored like
+                         * key.0 = value1
+                         * key.1 = value2
+                         * key.size = 2
+                         */
+                        preferences.keys.forEach { key ->
+                            if (key.startsWith(it.key + ".")) {
+                                preferences.remove(key)
+                            }
+                        }
+                    }
+                    Action.Clear -> preferences.clear()
                 }
             }
         }

CHANGELOG.md

@@ -1,32 +1,110 @@
+# Server: v0.7.0 + WebUI: r983
+## TL;DR
+- CBZ downloads support
+- Webview implementation based on Microsoft playwright, disabled for this release
+- Fixed compatibility with some chinese extensions
+- Support for Tachiyomi extensions lib 1.4
+- WebUI changes:
+    - Uhh, idk, find out yourself...
+
+## Tachidesk-Server Changelog
+- (r1159) v0.6.6 (by @AriaMoradi)
+- (r1160) add Chagelog TL;DR (by @AriaMoradi)
+- (r1161) fix Changelog typos (by @AriaMoradi)
+- (r1162) WebView based cloudflare interceptor ([#456](https://github.com/Suwayomi/Tachidesk-Server/pull/456) by @AriaMoradi)
+- (r1163) update issue mod (by @AriaMoradi)
+- (r1164) better description (by @AriaMoradi)
+- (r1165) fix regex (by @AriaMoradi)
+- (r1166) get default User Agent from WebView ([#457](https://github.com/Suwayomi/Tachidesk-Server/pull/457) by @AriaMoradi)
+- (r1167) implementation of android.graphics.BitmapFactory ([#460](https://github.com/Suwayomi/Tachidesk-Server/pull/460) by @animeavi)
+- (r1168) Basic android.graphics Rect and Canvas implementation ([#461](https://github.com/Suwayomi/Tachidesk-Server/pull/461) by @animeavi)
+- (r1169) Get Playwright working ([#462](https://github.com/Suwayomi/Tachidesk-Server/pull/462) by @Syer10)
+- (r1170) disable deb release (by @AriaMoradi)
+- (r1171) Fix debian release ([#463](https://github.com/Suwayomi/Tachidesk-Server/pull/463) by @mahor1221)
+- (r1172) Add better manga thumbnail handling ([#465](https://github.com/Suwayomi/Tachidesk-Server/pull/465) by @Syer10)
+- (r1173) Use extension list fallback if extensions fail to fetch ([#469](https://github.com/Suwayomi/Tachidesk-Server/pull/469) by @Syer10)
+- (r1174) fix when playwright fails on providing a UA (by @AriaMoradi)
+- (r1175) Update CategoryMetaTable.kt (by @AriaMoradi)
+- (r1176) fix CategoryMetaTable reference to CategoryTable ([#473](https://github.com/Suwayomi/Tachidesk-Server/pull/473) by @AriaMoradi)
+- (r1177) remove possibly misleading sentence (by @AriaMoradi)
+- (r1178) Clarify and Update (by @AriaMoradi)
+- (r1179) Clarify and Update (by @AriaMoradi)
+- (r1180) link to Tachiyomi section (by @AriaMoradi)
+- (r1181) fix typo (by @AriaMoradi)
+- (r1182) Improve Gradle Configuration ([#478](https://github.com/Suwayomi/Tachidesk-Server/pull/478) by @Syer10)
+- (r1183) Improve Playwright handling ([#479](https://github.com/Suwayomi/Tachidesk-Server/pull/479) by @Syer10)
+- (r1184) fix ambiguous reference issue on JDK 13+ (by @AriaMoradi)
+- (r1185) update gradle version (by @AriaMoradi)
+- (r1186) upgrade dorkbox stuff (by @AriaMoradi)
+- (r1187) Fixe Dex2Jar and dorkbox dependency issues ([#487](https://github.com/Suwayomi/Tachidesk-Server/pull/487) by @akabhirav)
+- (r1188) Fix logging and update system try ([#488](https://github.com/Suwayomi/Tachidesk-Server/pull/488) by @Syer10)
+- (r1189) add support for Extensions Lib 1.4 ([#496](https://github.com/Suwayomi/Tachidesk-Server/pull/496) by @Syer10)
+- (r1190) disable playwright for v0.6.7 (by @AriaMoradi)
+- (r1191) Decouple Cache and Download behaviour ([#493](https://github.com/Suwayomi/Tachidesk-Server/pull/493) by @akabhirav)
+- (r1192) rethink image cache ([#498](https://github.com/Suwayomi/Tachidesk-Server/pull/498) by @AriaMoradi)
+- (r1193) fix Page index issues for some providers ([#491](https://github.com/Suwayomi/Tachidesk-Server/pull/491) by @akabhirav)
+- (r1194) Download as CBZ ([#490](https://github.com/Suwayomi/Tachidesk-Server/pull/490) by @akabhirav)
+- (r1195) re-order config options (by @AriaMoradi)
+- (r1196) stop using depricated API (by @AriaMoradi)
+
+## Tachidesk-WebUI Changelog
+- (r964) Created a GridLayout enum and updated all locations to use it. ([#208](https://github.com/Suwayomi/Tachidesk-WebUI/pull/208) by @infix)
+- (r965) fix library update progress rendering ([#210](https://github.com/Suwayomi/Tachidesk-WebUI/pull/210) by @schroda)
+- (r966) Save reader settings per manga in Meta ([#216](https://github.com/Suwayomi/Tachidesk-WebUI/pull/216) by @schroda)
+- (r967) make default reader settings changeable ([#217](https://github.com/Suwayomi/Tachidesk-WebUI/pull/217) by @schroda)
+- (r968) [#211] Refresh Library after a update ([#212](https://github.com/Suwayomi/Tachidesk-WebUI/pull/212) by @schroda)
+- (r969) add logic for metadata migration ([#218](https://github.com/Suwayomi/Tachidesk-WebUI/pull/218) by @schroda)
+- (r970) set browser tab title ([#220](https://github.com/Suwayomi/Tachidesk-WebUI/pull/220) by @schroda)
+- (r971) Add tooltip containing full manga title to title of manga ([#221](https://github.com/Suwayomi/Tachidesk-WebUI/pull/221) by @schroda)
+- (r972) show more detailed upload dates for today and yesterday ([#222](https://github.com/Suwayomi/Tachidesk-WebUI/pull/222) by @schroda)
+- (r973) add GitHub action on pushing to run lint ([#224](https://github.com/Suwayomi/Tachidesk-WebUI/pull/224) by @schroda)
+- (r974) Ignore filters while searching ([#226](https://github.com/Suwayomi/Tachidesk-WebUI/pull/226) by @schroda)
+- (r975) force absolute import path ([#223](https://github.com/Suwayomi/Tachidesk-WebUI/pull/223) by @schroda)
+- (r976) add prettier for auto formatting ([#231](https://github.com/Suwayomi/Tachidesk-WebUI/pull/231) by @schroda)
+- (r977) Fix import path ([#228](https://github.com/Suwayomi/Tachidesk-WebUI/pull/228) by @schroda)
+- (r978) increase prettier line length to 120 ([#233](https://github.com/Suwayomi/Tachidesk-WebUI/pull/233) by @schroda)
+- (r979) Add chapter page dropdown ([#230](https://github.com/Suwayomi/Tachidesk-WebUI/pull/230) by @schroda)
+- (r980) add chapter dropdown to reader nav bar  ([#229](https://github.com/Suwayomi/Tachidesk-WebUI/pull/229) by @schroda)
+- (r981) Fix lint error ([#235](https://github.com/Suwayomi/Tachidesk-WebUI/pull/235) by @schroda)
+- (r982) Fix reader nav bar scroll to page ([#236](https://github.com/Suwayomi/Tachidesk-WebUI/pull/236) by @schroda)
+- (r964) Created a GridLayout enum and updated all locations to use it. ([#208](https://github.com/Suwayomi/Tachidesk-WebUI/pull/208) by @infix)
+
+
+
 # Server: v0.6.6 + WebUI: r963
 ## TL;DR
-- N/A
+- Batch actions for chapters
+- Improved the downloader
+- WebUI changes:
+    - Support for chapter actions
+    - a lot of code cleanup
+    - some bugfixes
 
 ## Tachidesk-Server Changelog
 - (r1114) fix broken links (by @AriaMoradi)
 - (r1115) fix more broken stuff (by @AriaMoradi)
 - (r1116) fix more broken stuff (by @AriaMoradi)
 - (r1117) fix more broken stuff (by @AriaMoradi)
-- (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393)) 83997633+vedantmgoyal2009@users.noreply.github.com
+- (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393) by @vedantmgoyal2009)
-- (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396)) 30526233+voltrare@users.noreply.github.com
+- (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396) by @vedantmgoyal2009)
-- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397)) mahor1221@pm.me
+- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397) by @mahor1221)
 - (r1121) bump version (by @AriaMoradi)
 - (r1122) Update Changelog (by @AriaMoradi)
-- (r1123) Add libc++-dev ([#405](https://github.com/Suwayomi/Tachidesk-Server/pull/405)) mahor1221@pm.me
+- (r1123) Add libc++-dev ([#405](https://github.com/Suwayomi/Tachidesk-Server/pull/405) by @mahor1221)
-- (r1124) Revert back to correct way of handling jre_dir ([#408](https://github.com/Suwayomi/Tachidesk-Server/pull/408)) mahor1221@pm.me
+- (r1124) Revert back to correct way of handling jre_dir ([#408](https://github.com/Suwayomi/Tachidesk-Server/pull/408) by @mahor1221)
-- (r1125) Update winget.yml ([#410](https://github.com/Suwayomi/Tachidesk-Server/pull/410)) 83997633+vedantmgoyal2009@users.noreply.github.com
+- (r1125) Update winget.yml ([#410](https://github.com/Suwayomi/Tachidesk-Server/pull/410) by @vedantmgoyal2009)
-- (r1126) Remove support for Sorayomi web interface ([#414](https://github.com/Suwayomi/Tachidesk-Server/pull/414)) ebbinghaus.marco@gmail.com
+- (r1126) Remove support for Sorayomi web interface ([#414](https://github.com/Suwayomi/Tachidesk-Server/pull/414) by @marcoebbinghaus)
 - (r1127) Fix downloader memory leak ([#418](https://github.com/Suwayomi/Tachidesk-Server/pull/418) by @Syer10)
 - (r1128) Documentation cleanup ([#417](https://github.com/Suwayomi/Tachidesk-Server/pull/417) by @Syer10)
 - (r1129) Updater cleanup and improvements ([#416](https://github.com/Suwayomi/Tachidesk-Server/pull/416) by @Syer10)
-- (r1130) replace quickjs with Mozilla Rhino ([#415](https://github.com/Suwayomi/Tachidesk-Server/pull/415)) 747367352@qq.com
+- (r1130) replace quickjs with Mozilla Rhino ([#415](https://github.com/Suwayomi/Tachidesk-Server/pull/415) by @xhzhe)
 - (r1131) ktlint (by @AriaMoradi)
 - (r1132) move Tachiyomi's BuildConfig to kotlin dir (by @AriaMoradi)
 - (r1133) remove BuildConfig as extensions now use AppInfo (by @AriaMoradi)
 - (r1134) include list of mangas missing source in restore report ([#421](https://github.com/Suwayomi/Tachidesk-Server/pull/421) by @AriaMoradi)
 - (r1135) Update dependencies ([#422](https://github.com/Suwayomi/Tachidesk-Server/pull/422) by @Syer10)
 - (r1136) Lint ([#423](https://github.com/Suwayomi/Tachidesk-Server/pull/423) by @Syer10)
-- (r1137) Fix: Error handling for popular/latest api if pageNum was supplied as zero ([#424](https://github.com/Suwayomi/Tachidesk-Server/pull/424)) anurag4884@gmail.com
+- (r1137) Fix: Error handling for popular/latest api if pageNum was supplied as zero ([#424](https://github.com/Suwayomi/Tachidesk-Server/pull/424) by @meta-boy)
 - (r1138) Add cache control header to manga page response ([#430](https://github.com/Suwayomi/Tachidesk-Server/pull/430) by @martinek)
 - (r1139) add MangaTable.lastFetchedAt and ChapterTable.chaptersLastFetchedAt ([#431](https://github.com/Suwayomi/Tachidesk-Server/pull/431) by @martinek)
 - (r1140) Pre-load meta entries for all chapters for optimization ([#432](https://github.com/Suwayomi/Tachidesk-Server/pull/432) by @martinek)
@@ -82,7 +160,7 @@
 - (r1117) fix more broken stuff (by @AriaMoradi)
 - (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393) by @vedantmgoyal2009)
 - (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396) by @voltrare)
-- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397)) by @mahor1221)
+- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397) by @mahor1221)
 - (r1121) bump version (by @AriaMoradi)
 
 ## Tachidesk-WebUI Changelog

CONTRIBUTING.md

@@ -2,9 +2,10 @@
 ## Where should I start?
 Checkout [This Kanban Board](https://github.com/Suwayomi/Tachidesk/projects/1) to see the rough development roadmap.
 
-**Note 1:** Notify the developers on [Suwayomi discord](https://discord.gg/DDZdqZWaHA) (#tachidesk-server and #tachidesk-webui channels) or open a WIP pull request before starting if you decide to take on working on anything from/not from the roadmap in order to avoid parallel efforts on the same issue/feature.
+### Important notes
-
+- Notify the developers on [Suwayomi discord](https://discord.gg/DDZdqZWaHA) (#tachidesk-server and #tachidesk-webui channels) or open a WIP pull request before starting if you decide to take on working on anything from/not from the roadmap in order to avoid parallel efforts on the same issue/feature.
-**Note 2:** Your pull request will be squashed into a single commit.
+- Your pull request will be squashed into a single commit.
+- We hate big pull requests, make them as small as possible, change one meaningful thing. Spam pull requests, we don't mind.
 
 ### Project goals and vision
 - Porting Tachiyomi and covering it's features

README.md

@@ -34,25 +34,22 @@ A free and open source manga reader server that runs extensions built for [Tachi
 
 Tachidesk is an independent Tachiyomi compatible software and is **not a Fork of** Tachiyomi.
 
-`Tachidesk` is a general term used to describe the combination of Tachidesk-Server(this project) and one of our clients. 
-Think of it roughly like the concept of "distribution" in GNU/Linux distributions, in which Linux(Tachidesk-Server) is the kernel and the difference is which desktop environment(Tachidesk client) you get with it. 
-
 Tachidesk-Server is as multi-platform as you can get. Any platform that runs java and/or has a modern browser can run it. This includes Windows, Linux, macOS, chrome OS, etc. Follow [Downloading and Running the app](#downloading-and-running-the-app) for installation instructions.
 
-Ability to sync with Tachiyomi is a planned feature.
+Ability to sync with Tachiyomi is a planned feature, for more info look [here](#syncing-with-tachiyomi).
 
 # Tachidesk client projects
-**You need a client/user interface app as a front-end for Tachidesk-Server, if you Directly Download Tachidesk-Server you'll get a bundled version of [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI) with it.**
+**You need a client/user interface app as a front-end for Tachidesk-Server, if you [Directly Download Tachidesk-Server](https://github.com/Suwayomi/Tachidesk-Server/releases/latest) you'll get a bundled version of [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI) with it.**
 
 Here's a list of known clients/user interfaces for Tachidesk-Server:
 ##### Actively Developed Cients
-- [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI): The web/ElectronJS front-end that Tachidesk-Server is traditionally shipped with. Usually gets new features faster.
+- [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI): The web/ElectronJS front-end that Tachidesk-Server ships with by default.
 - [Tachidesk-JUI](https://github.com/Suwayomi/Tachidesk-JUI): The native desktop front-end for Tachidesk-Server. Currently the most advanced.
-- [Tachidesk-qtui](https://github.com/Suwayomi/Tachidesk-qtui): A C++/Qt front-end for mobile devices(Android/linux), in super early stage of development.
+- [Tachidesk-qtui](https://github.com/Suwayomi/Tachidesk-qtui): A C++/Qt front-end for mobile devices(Android/linux), feature support is basic.
-- [Tachidesk-Sorayomi](https://github.com/Suwayomi/Tachidesk-Sorayomi): A Flutter front-end for Desktop(Linux, windows, etc.), Web and Android. UI and UX similar to Tachiyomi.
+- [Tachidesk-Sorayomi](https://github.com/Suwayomi/Tachidesk-Sorayomi): A Flutter front-end for Desktop(Linux, windows, etc.), Web and Android with a User Inerface inspired by Tachiyomi.
 ##### Inctive/Abandoned Cients
-- [Equinox](https://github.com/Suwayomi/Equinox): A web user interface made with Vue.js, in super early stage of development.
+- [Equinox](https://github.com/Suwayomi/Equinox): A web user interface made with Vue.js.
-- [Tachidesk-GTK](https://github.com/mahor1221/Tachidesk-GTK): A native Rust/GTK desktop client, in super early stage of development.
+- [Tachidesk-GTK](https://github.com/mahor1221/Tachidesk-GTK): A native Rust/GTK desktop client.
 
 ## Is this application usable? Should I test it?
 Here is a list of current features:
@@ -180,3 +177,7 @@ Changes to both codebases is licensed under `MPL v. 2.0` as the rest of this pro
     This Source Code Form is subject to the terms of the Mozilla Public
     License, v. 2.0. If a copy of the MPL was not distributed with this
     file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+## Disclaimer
+
+The developer of this application does not have any affiliation with the content providers available.

build.gradle.kts

@@ -1,13 +1,14 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile
 import org.jmailen.gradle.kotlinter.tasks.FormatTask
 import org.jmailen.gradle.kotlinter.tasks.LintTask
 
+@Suppress("DSL_SCOPE_VIOLATION")
 plugins {
-    kotlin("jvm") version kotlinVersion
+    alias(libs.plugins.kotlin.jvm)
-    kotlin("plugin.serialization") version kotlinVersion
+    alias(libs.plugins.kotlin.serialization)
-    id("org.jmailen.kotlinter") version "3.12.0"
+    alias(libs.plugins.kotlinter)
-    id("com.github.gmazzo.buildconfig") version "3.1.0" apply false
+    alias(libs.plugins.buildconfig) apply false
-    id("de.undercouch.download") version "5.3.0"
+    alias(libs.plugins.download)
 }
 
 allprojects {
@@ -18,30 +19,22 @@ allprojects {
     repositories {
         mavenCentral()
         google()
-        maven("https://jitpack.io")
         maven("https://github.com/Suwayomi/Tachidesk-Server/raw/android-jar/")
+        maven("https://jitpack.io")
     }
 }
 
-val projects = listOf(
+subprojects {
-        project(":AndroidCompat"),
+    plugins.withType<JavaPlugin> {
-        project(":AndroidCompat:Config"),
+        extensions.configure<JavaPluginExtension> {
-        project(":server")
+            sourceCompatibility = JavaVersion.VERSION_1_8
-)
+            targetCompatibility = JavaVersion.VERSION_1_8
-
+        }
-configure(projects) {
-    apply(plugin = "org.jetbrains.kotlin.jvm")
-    apply(plugin = "org.jetbrains.kotlin.plugin.serialization")
-    apply(plugin = "org.jmailen.kotlinter")
-
-    java {
-        sourceCompatibility = JavaVersion.VERSION_1_8
-        targetCompatibility = JavaVersion.VERSION_1_8
     }
 
     tasks {
-        withType<KotlinCompile> {
+        withType<KotlinJvmCompile> {
-            dependsOn(formatKotlin)
+            dependsOn("formatKotlin")
             kotlinOptions {
                 jvmTarget = JavaVersion.VERSION_1_8.toString()
             }
@@ -55,58 +48,4 @@ configure(projects) {
             source(files("src/kotlin"))
         }
     }
-
-
-    dependencies {
-        // Kotlin
-        implementation(kotlin("stdlib-jdk8"))
-        implementation(kotlin("reflect"))
-        testImplementation(kotlin("test-junit5"))
-
-        // coroutines
-        val coroutinesVersion = "1.6.4"
-        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
-        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:$coroutinesVersion")
-        testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion")
-
-        val kotlinSerializationVersion = "1.4.1"
-        implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinSerializationVersion")
-        implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:$kotlinSerializationVersion")
-
-        // Dependency Injection
-        implementation("org.kodein.di:kodein-di-conf-jvm:7.15.0")
-
-        // Logging
-        // Stuck on old versions since
-        // 1. Logback 1.3.0+ requires Java 9
-        // 2. Slf4j 2.0.0+ doesn't register older versions of Logback
-        // 3. Kotlin-logging 3.0.2+ requires Java 11, but this is probably a bug
-        implementation("org.slf4j:slf4j-api:1.7.32")
-        implementation("ch.qos.logback:logback-classic:1.2.6")
-        implementation("io.github.microutils:kotlin-logging:2.1.21")
-
-        // ReactiveX
-        implementation("io.reactivex:rxjava:1.3.8")
-
-        // dependency both in AndroidCompat and extensions, version locked by Tachiyomi app/extensions
-        implementation("org.jsoup:jsoup:1.15.3")
-
-        // dependency of :AndroidCompat:Config
-        implementation("com.typesafe:config:1.4.2")
-        implementation("io.github.config4k:config4k:0.5.0")
-
-        // to get application content root
-        implementation("net.harawata:appdirs:1.2.1")
-
-        // dex2jar
-        val dex2jarVersion = "v56"
-        implementation("com.github.ThexXTURBOXx.dex2jar:dex-translator:$dex2jarVersion")
-        implementation("com.github.ThexXTURBOXx.dex2jar:dex-tools:$dex2jarVersion")
-
-        // APK parser
-        implementation("net.dongliu:apk-parser:2.6.10")
-
-        // dependency both in AndroidCompat and server, version locked by javalin
-        implementation("com.fasterxml.jackson.core:jackson-annotations:2.12.4")
-    }
 }

buildSrc/src/main/kotlin/Constants.kt

@@ -7,20 +7,18 @@ import java.io.BufferedReader
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-const val kotlinVersion = "1.7.20"
-
 const val MainClass = "suwayomi.tachidesk.MainKt"
 
 // should be bumped with each stable release
-val tachideskVersion = System.getenv("ProductVersion") ?: "v0.6.6"
+val tachideskVersion = System.getenv("ProductVersion") ?: "v0.7.0"
 
-val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r963"
+val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r1045"
 
 // counts commits on the master branch
 val tachideskRevision = runCatching {
-    System.getenv("ProductRevision") ?: Runtime
+    System.getenv("ProductRevision") ?: ProcessBuilder()
-        .getRuntime()
+        .command("git", "rev-list", "HEAD", "--count")
-        .exec("git rev-list HEAD --count")
+        .start()
         .let { process ->
             process.waitFor()
             val output = process.inputStream.use {

gradle/libs.versions.toml

@@ -0,0 +1,212 @@
+[versions]
+kotlin = "1.8.0"
+coroutines = "1.6.4"
+serialization = "1.4.1"
+okhttp = "5.0.0-alpha.11" # Major version is locked by Tachiyomi extensions
+javalin = "4.6.6" # Javalin 5.0.0+ requires Java 11
+jackson = "2.13.3" # jackson version locked by javalin, ref: `io.javalin.core.util.OptionalDependency`
+exposed = "0.40.1"
+dex2jar = "v60"
+rhino = "1.7.14"
+settings = "1.0.0-RC"
+twelvemonkeys = "3.9.4"
+
+[libraries]
+# Kotlin
+kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" }
+kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" }
+kotlin-test-junit5 = { module = "org.jetbrains.kotlin:kotlin-test-junit5", version.ref = "kotlin" }
+
+# Coroutines
+coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" }
+coroutines-jdk8 = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-jdk8", version.ref = "coroutines" }
+coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" }
+
+# Serialization
+serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" }
+serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "serialization" }
+
+# Logging
+slf4japi = "org.slf4j:slf4j-api:2.0.6"
+logback = "ch.qos.logback:logback-classic:1.3.5"
+kotlinlogging = "io.github.microutils:kotlin-logging:3.0.5"
+
+# OkHttp
+okhttp-core = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
+okhttp-logging = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" }
+okhttp-dnsoverhttps = { module = "com.squareup.okhttp3:okhttp-dnsoverhttps", version.ref = "okhttp" }
+okio = "com.squareup.okio:okio:3.3.0"
+
+# Javalin api
+javalin-core = { module = "io.javalin:javalin", version.ref = "javalin" }
+javalin-openapi = { module = "io.javalin:javalin-openapi", version.ref = "javalin" }
+jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" }
+jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" }
+jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" }
+
+# Exposed ORM
+exposed-core = { module = "org.jetbrains.exposed:exposed-core", version.ref = "exposed" }
+exposed-dao = { module = "org.jetbrains.exposed:exposed-dao", version.ref = "exposed" }
+exposed-jdbc = { module = "org.jetbrains.exposed:exposed-jdbc", version.ref = "exposed" }
+exposed-javatime = { module = "org.jetbrains.exposed:exposed-java-time", version.ref = "exposed" }
+h2 = "com.h2database:h2:1.4.200" # current database driver, can't update to h2 v2 without sql migration
+
+# Exposed Migrations
+exposed-migrations = "com.github.Suwayomi:exposed-migrations:3.2.0"
+
+# Dependency Injection
+kodein = "org.kodein.di:kodein-di-conf-jvm:7.15.0"
+
+# tray icon
+systemtray-core = "com.dorkbox:SystemTray:4.2.1"
+systemtray-utils = "com.dorkbox:Utilities:1.39" # version locked by SystemTray
+systemtray-desktop = "com.dorkbox:Desktop:1.0"
+
+# dependencies of Tachiyomi extensions
+injekt = "com.github.inorichi.injekt:injekt-core:65b0440"
+rxjava = "io.reactivex:rxjava:1.3.8"
+jsoup = "org.jsoup:jsoup:1.15.3"
+
+# Config
+config = "com.typesafe:config:1.4.2"
+config4k = "io.github.config4k:config4k:0.5.0"
+
+# Sort
+sort = "com.github.gpanther:java-nat-sort:natural-comparator-1.1"
+
+# Android stub library
+android-stubs = "com.github.Suwayomi:android-jar:1.0.0"
+
+# Asm modificiation
+asm = "org.ow2.asm:asm:9.4" # version locked by Dex2Jar
+dex2jar-translator = { module = "com.github.ThexXTURBOXx.dex2jar:dex-translator", version.ref = "dex2jar" }
+dex2jar-tools = { module = "com.github.ThexXTURBOXx.dex2jar:dex-tools", version.ref = "dex2jar" }
+
+# APK
+apk-parser = "net.dongliu:apk-parser:2.6.10"
+apksig = "com.android.tools.build:apksig:7.2.1"
+
+# Xml
+xmlpull = "xmlpull:xmlpull:1.1.3.4a"
+
+# Disk & File
+appdirs = "net.harawata:appdirs:1.2.1"
+zip4j = "net.lingala.zip4j:zip4j:2.11.2"
+junrar = "com.github.junrar:junrar:7.5.3"
+
+# AES/CBC/PKCS7Padding Cypher provider
+bouncycastle = "org.bouncycastle:bcprov-jdk18on:1.72"
+
+# AndroidX annotations
+android-annotations = "androidx.annotation:annotation:1.5.0"
+
+# Substitute for duktape-android
+rhino-runtime = { module = "org.mozilla:rhino-runtime", version.ref = "rhino" }  # slimmer version of 'org.mozilla:rhino'
+rhino-engine = { module = "org.mozilla:rhino-engine", version.ref = "rhino" } # provides the same interface as 'javax.script' a.k.a Nashorn
+
+# Settings
+settings-core = { module = "com.russhwolf:multiplatform-settings-jvm", version.ref = "settings" }
+settings-serialization = { module = "com.russhwolf:multiplatform-settings-serialization-jvm", version.ref = "settings" }
+
+# ICU4J
+icu4j = "com.ibm.icu:icu4j:72.1"
+
+# Image Decoding implementation provider
+twelvemonkeys-common-lang = { module = "com.twelvemonkeys.common:common-lang", version.ref = "twelvemonkeys" }
+twelvemonkeys-common-io = { module = "com.twelvemonkeys.common:common-io", version.ref = "twelvemonkeys" }
+twelvemonkeys-common-image = { module = "com.twelvemonkeys.common:common-image", version.ref = "twelvemonkeys" }
+twelvemonkeys-imageio-core = { module = "com.twelvemonkeys.imageio:imageio-core", version.ref = "twelvemonkeys" }
+twelvemonkeys-imageio-metadata = { module = "com.twelvemonkeys.imageio:imageio-metadata", version.ref = "twelvemonkeys" }
+twelvemonkeys-imageio-jpeg = { module = "com.twelvemonkeys.imageio:imageio-jpeg", version.ref = "twelvemonkeys" }
+twelvemonkeys-imageio-webp = { module = "com.twelvemonkeys.imageio:imageio-webp", version.ref = "twelvemonkeys" }
+
+# Testing
+mockk = "io.mockk:mockk:1.13.2"
+
+[plugins]
+# Kotlin
+kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"}
+kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin"}
+
+# Linter
+kotlinter = { id = "org.jmailen.kotlinter", version = "3.12.0"}
+
+# Build config
+buildconfig = { id = "com.github.gmazzo.buildconfig", version = "3.1.0"}
+
+# Download
+download = { id = "de.undercouch.download", version = "5.3.0"}
+
+# ShadowJar
+shadowjar = { id = "com.github.johnrengelman.shadow", version = "7.1.2"}
+
+[bundles]
+shared = [
+    "kotlin-stdlib-jdk8",
+    "kotlin-reflect",
+    "coroutines-core",
+    "coroutines-jdk8",
+    "serialization-json",
+    "serialization-protobuf",
+    "kodein",
+    "slf4japi",
+    "logback",
+    "kotlinlogging",
+    "appdirs",
+    "rxjava",
+    "jsoup",
+    "config",
+    "config4k",
+    "dex2jar-translator",
+    "dex2jar-tools",
+    "apk-parser",
+    "jackson-annotations"
+]
+
+sharedTest = [
+    "kotlin-test-junit5",
+    "coroutines-test",
+]
+
+okhttp = [
+    "okhttp-core",
+    "okhttp-logging",
+    "okhttp-dnsoverhttps",
+]
+javalin = [
+    "javalin-core",
+    "javalin-openapi",
+]
+jackson = [
+    "jackson-databind",
+    "jackson-kotlin",
+    "jackson-annotations",
+]
+exposed = [
+    "exposed-core",
+    "exposed-dao",
+    "exposed-jdbc",
+    "exposed-javatime",
+]
+systemtray = [
+    "systemtray-core",
+    "systemtray-utils",
+    "systemtray-desktop"
+]
+rhino = [
+    "rhino-runtime",
+    "rhino-engine",
+]
+settings = [
+    "settings-core",
+    "settings-serialization",
+]
+twelvemonkeys = [
+    "twelvemonkeys-common-lang",
+    "twelvemonkeys-common-io",
+    "twelvemonkeys-common-image",
+    "twelvemonkeys-imageio-core",
+    "twelvemonkeys-imageio-metadata",
+    "twelvemonkeys-imageio-jpeg",
+    "twelvemonkeys-imageio-webp",
+]

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 #
-# Copyright © 2015-2021 the original authors.
+# Copyright © 2015-2021 the original authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,10 +32,10 @@
 #       Busybox and similar reduced shells will NOT work, because this script
 #       requires all of these POSIX shell features:
 #         * functions;
-#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
-#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
-#         * compound commands having a testable exit status, especially «case»;
+#         * compound commands having a testable exit status, especially «case»;
-#         * various built-in commands including «command», «set», and «ulimit».
+#         * various built-in commands including «command», «set», and «ulimit».
 #
 #   Important for patching:
 #
@@ -205,6 +205,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,7 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+set EXIT_CODE=%ERRORLEVEL%
-exit /b 1
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

scripts/bundler.sh

@@ -57,6 +57,9 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
+      PLAYWRIGHT_PLATFORM="linux"
+      setup_playwright
+
       RELEASE="$RELEASE_NAME.tar.gz"
       make_linux_bundle
       move_release_to_output_dir
@@ -70,6 +73,9 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
+      PLAYWRIGHT_PLATFORM="mac"
+      setup_playwright
+
       RELEASE="$RELEASE_NAME.zip"
       make_macos_bundle
       move_release_to_output_dir
@@ -83,6 +89,9 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
+      PLAYWRIGHT_PLATFORM="mac-arm64"
+      setup_playwright
+
       RELEASE="$RELEASE_NAME.zip"
       make_macos_bundle
       move_release_to_output_dir
@@ -96,6 +105,9 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
+      PLAYWRIGHT_PLATFORM="win64"
+      setup_playwright
+
       RELEASE="$RELEASE_NAME.zip"
       make_windows_bundle
       move_release_to_output_dir
@@ -113,6 +125,10 @@ main() {
       ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON"
       download_jre_and_electron
 
+      PYTHON=Winpython64-3.10.9.0dot.exe
+      PYTHON_URL=https://github.com/winpython/winpython/releases/download/5.3.20221233/Winpython64-3.10.9.0dot.exe
+      setup_undetected_chromedriver_and_python
+
       RELEASE="$RELEASE_NAME.zip"
       make_windows_bundle
       move_release_to_output_dir
@@ -268,6 +284,25 @@ make_windows_package() {
     "$RELEASE_NAME/jre.wxs" "$RELEASE_NAME/electron.wxs" -o "$RELEASE"
 }
 
+setup_python() {
+  mkdir "$RELEASE_NAME/"
+  curl -L "$PYTHON_URL" -o "$PYTHON"
+  7z x $PYTHON
+  mv WPy64-31090/python-3.10.9.amd64 "$RELEASE_NAME/python"
+}
+
+setup_undetected_chromedriver() {
+  curl -L "https://github.com/Suwayomi/undetected-chromedriver/archive/refs/heads/master.zip" -o undetected-chromedriver-master.zip
+  unzip undetected-chromedriver-master.zip
+  mv undetected-chromedriver-master "$RELEASE_NAME/undetected-chromedriver"
+}
+
+
+setup_undetected_chromedriver_and_python() {
+  setup_python
+  setup_undetected_chromedriver
+}
+
 # Error handler
 # set -u: Treat unset variables as an error when substituting.
 # set -o pipefail: Prevents errors in pipeline from being masked.

scripts/resources/deb/control

@@ -2,7 +2,7 @@ Source: tachidesk-server
 Section: web
 Priority: optional
 Maintainer: Mahor1221 <mahor1221@pm.me>
-Build-Depends: debhelper-compat (= 12), dh-exec
+Build-Depends: debhelper-compat (= 13), dh-exec
 Standards-Version: 4.5.1
 Homepage: https://github.com/Suwayomi/Tachidesk-Server
 

server/build.gradle.kts

@@ -1,80 +1,67 @@
 import de.undercouch.gradle.tasks.download.Download
 import java.time.Instant
 
+@Suppress("DSL_SCOPE_VIOLATION")
 plugins {
+    id(libs.plugins.kotlin.jvm.get().pluginId)
+    id(libs.plugins.kotlin.serialization.get().pluginId)
+    id(libs.plugins.kotlinter.get().pluginId)
     application
-    id("com.github.johnrengelman.shadow") version "7.1.2"
+    alias(libs.plugins.shadowjar)
-    id("com.github.gmazzo.buildconfig")
+    id(libs.plugins.buildconfig.get().pluginId)
 }
 
 dependencies {
-    // okhttp
+    // Shared
-    val okhttpVersion = "4.10.0" // Major version is locked by Tachiyomi extensions
+    implementation(libs.bundles.shared)
-    implementation("com.squareup.okhttp3:okhttp:$okhttpVersion")
+    testImplementation(libs.bundles.sharedTest)
-    implementation("com.squareup.okhttp3:logging-interceptor:$okhttpVersion")
+
-    implementation("com.squareup.okhttp3:okhttp-dnsoverhttps:$okhttpVersion")
+    // OkHttp
-    implementation("com.squareup.okio:okio:3.2.0")
+    implementation(libs.bundles.okhttp)
+    implementation(libs.okio)
 
     // Javalin api
-    // Javalin 5.0.0+ requires Java 11
+    implementation(libs.bundles.javalin)
-    implementation("io.javalin:javalin:4.6.6")
+    implementation(libs.bundles.jackson)
-    implementation("io.javalin:javalin-openapi:4.6.6")
-    // jackson version locked by javalin, ref: `io.javalin.core.util.OptionalDependency`
-    val jacksonVersion = "2.13.3"
-    implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
-    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
 
     // Exposed ORM
-    val exposedVersion = "0.40.1"
+    implementation(libs.bundles.exposed)
-    implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
+    implementation(libs.h2)
-    implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
-    implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
-    implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
-    // current database driver, can't update to h2 v2 without sql migration
-    implementation("com.h2database:h2:1.4.200")
 
     // Exposed Migrations
-    implementation("com.github.Suwayomi:exposed-migrations:3.2.0")
+    implementation(libs.exposed.migrations)
 
     // tray icon
-    implementation("com.dorkbox:SystemTray:4.1")
+    implementation(libs.bundles.systemtray)
-    implementation("com.dorkbox:Utilities:1.9") // version locked by SystemTray
 
     // dependencies of Tachiyomi extensions, some are duplicate, keeping it here for reference
-    implementation("com.github.inorichi.injekt:injekt-core:65b0440")
+    implementation(libs.injekt)
-    implementation("com.squareup.okhttp3:okhttp:$okhttpVersion")
+    implementation(libs.okhttp.core)
-    implementation("io.reactivex:rxjava:1.3.8")
+    implementation(libs.rxjava)
-    implementation("org.jsoup:jsoup:1.15.3")
+    implementation(libs.jsoup)
 
     // Sort
-    implementation("com.github.gpanther:java-nat-sort:natural-comparator-1.1")
+    implementation(libs.sort)
 
     // asm for ByteCodeEditor(fixing SimpleDateFormat) (must match Dex2Jar version)
-    implementation("org.ow2.asm:asm:9.4")
+    implementation(libs.asm)
 
     // Disk & File
-    implementation("net.lingala.zip4j:zip4j:2.11.2")
+    implementation(libs.zip4j)
-    implementation("com.github.junrar:junrar:7.5.3")
+    implementation(libs.junrar)
-
-    // CloudflareInterceptor
-    implementation("net.sourceforge.htmlunit:htmlunit:2.65.1")
 
     // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga
-    implementation("org.bouncycastle:bcprov-jdk18on:1.72")
+    implementation(libs.bouncycastle)
-
-    // Source models and interfaces from Tachiyomi 1.x
-    // using source class from tachiyomi commit 9493577de27c40ce8b2b6122cc447d025e34c477 to not depend on tachiyomi.sourceapi
-//    implementation("tachiyomi.sourceapi:source-api:1.1")
 
     // AndroidCompat
-    implementation(project(":AndroidCompat"))
+    implementation(projects.androidCompat)
-    implementation(project(":AndroidCompat:Config"))
+    implementation(projects.androidCompat.config)
 
     // uncomment to test extensions directly
 //    implementation(fileTree("lib/"))
     implementation(kotlin("script-runtime"))
 
-    testImplementation("io.mockk:mockk:1.13.2")
+    testImplementation(libs.mockk)
 }
 
 application {

server/src/main/kotlin/eu/kanade/tachiyomi/AppModule.kt

@@ -16,6 +16,7 @@ package eu.kanade.tachiyomi
 // import eu.kanade.tachiyomi.data.track.TrackManager
 // import eu.kanade.tachiyomi.extension.ExtensionManager
 import android.app.Application
+import eu.kanade.tachiyomi.network.JavaScriptEngine
 import eu.kanade.tachiyomi.network.NetworkHelper
 import kotlinx.serialization.json.Json
 import rx.Observable
@@ -41,6 +42,8 @@ class AppModule(val app: Application) : InjektModule {
 
         addSingletonFactory { NetworkHelper(app) }
 
+        addSingletonFactory { JavaScriptEngine(app) }
+
 //        addSingletonFactory { SourceManager(app).also { get<ExtensionManager>().init(it) } }
 //
 //        addSingletonFactory { ExtensionManager(app) }

server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt

@@ -0,0 +1,27 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import app.cash.quickjs.QuickJs
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+/**
+ * Util for evaluating JavaScript in sources.
+ */
+class JavaScriptEngine(context: Context) {
+
+    /**
+     * Evaluate arbitrary JavaScript code and get the result as a primitive type
+     * (e.g., String, Int).
+     *
+     * @since extensions-lib 1.4
+     * @param script JavaScript to execute.
+     * @return Result of JavaScript code as a primitive type.
+     */
+    @Suppress("UNUSED", "UNCHECKED_CAST")
+    suspend fun <T> evaluate(script: String): T = withContext(Dispatchers.IO) {
+        QuickJs.create().use {
+            it.evaluate(script) as T
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -39,6 +39,7 @@ class NetworkHelper(context: Context) {
                 .cookieJar(cookieManager)
                 .connectTimeout(30, TimeUnit.SECONDS)
                 .readTimeout(30, TimeUnit.SECONDS)
+                .callTimeout(2, TimeUnit.MINUTES)
                 .addInterceptor(UserAgentInterceptor())
 
             if (serverConfig.debugLogsEnabled) {

server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt

@@ -62,7 +62,7 @@ suspend fun Call.await(): Response {
             object : Callback {
                 override fun onResponse(call: Call, response: Response) {
                     if (!response.isSuccessful) {
-                        continuation.resumeWithException(Exception("HTTP error ${response.code}"))
+                        continuation.resumeWithException(HttpException(response.code))
                         return
                     }
 
@@ -94,7 +94,7 @@ fun Call.asObservableSuccess(): Observable<Response> {
         .doOnNext { response ->
             if (!response.isSuccessful) {
                 response.close()
-                throw Exception("HTTP error ${response.code}")
+                throw HttpException(response.code)
             }
         }
 }
@@ -136,3 +136,5 @@ inline fun <reified T> Response.parseAs(): T {
         return json.decodeFromString(responseBody)
     }
 }
+
+class HttpException(val code: Int) : IllegalStateException("HTTP error $code")

server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.network
 import okhttp3.CacheControl
 import okhttp3.FormBody
 import okhttp3.Headers
+import okhttp3.HttpUrl
 import okhttp3.Request
 import okhttp3.RequestBody
 import java.util.concurrent.TimeUnit.MINUTES
@@ -23,6 +24,21 @@ fun GET(
         .build()
 }
 
+/**
+ * @since extensions-lib 1.4
+ */
+fun GET(
+    url: HttpUrl,
+    headers: Headers = DEFAULT_HEADERS,
+    cache: CacheControl = DEFAULT_CACHE_CONTROL
+): Request {
+    return Request.Builder()
+        .url(url)
+        .headers(headers)
+        .cacheControl(cache)
+        .build()
+}
+
 fun POST(
     url: String,
     headers: Headers = DEFAULT_HEADERS,

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -1,19 +1,30 @@
 package eu.kanade.tachiyomi.network.interceptor
 
-import com.gargoylesoftware.htmlunit.BrowserVersion
-import com.gargoylesoftware.htmlunit.WebClient
-import com.gargoylesoftware.htmlunit.html.HtmlPage
 import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.resolveWithWebView
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import mu.KotlinLogging
 import okhttp3.Cookie
 import okhttp3.HttpUrl
 import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
+import java.io.BufferedReader
+import java.io.Closeable
+import java.io.File
 import java.io.IOException
+import java.io.InputStreamReader
+import java.io.PrintWriter
+import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
-// from TachiWeb-Server
 class CloudflareInterceptor : Interceptor {
     private val logger = KotlinLogging.logger {}
 
@@ -25,20 +36,26 @@ class CloudflareInterceptor : Interceptor {
 
         logger.trace { "CloudflareInterceptor is being used." }
 
-        val response = chain.proceed(originalRequest)
+        val originalResponse = chain.proceed(chain.request())
 
         // Check if Cloudflare anti-bot is on
-        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
+        if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) {
-            return response
+            return originalResponse
         }
 
         logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
 
+        if (!serverConfig.webviewEnabled) {
+            throw CloudflareBypassException("Webview is disabled, enable it in server config")
+        }
+
         return try {
-            response.close()
+            originalResponse.close()
             network.cookies.remove(originalRequest.url.toUri())
 
-            chain.proceed(resolveChallenge(response))
+            val request = resolveWithWebView(originalRequest)
+
+            chain.proceed(request)
         } catch (e: Exception) {
             // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
             // we don't crash the entire app
@@ -46,65 +63,282 @@ class CloudflareInterceptor : Interceptor {
         }
     }
 
-    private fun resolveChallenge(response: Response): Request {
+    companion object {
-        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
+        private val ERROR_CODES = listOf(403, 503)
-            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
+        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-            .build()
+        private val COOKIE_NAMES = listOf("cf_clearance")
-        val convertedCookies = WebClient(browserVersion).use { webClient ->
+    }
-            webClient.options.isThrowExceptionOnFailingStatusCode = false
+}
-            webClient.options.isThrowExceptionOnScriptError = false
+
-            webClient.getPage<HtmlPage>(response.request.url.toString())
+object CloudflareBypasser {
-            webClient.waitForBackgroundJavaScript(10000)
+    private val logger = KotlinLogging.logger {}
-            // Challenge solved, process cookies
+    private val network: NetworkHelper by injectLazy()
-            webClient.cookieManager.cookies.filter {
+
-                // Only include Cloudflare cookies
+    fun resolveWithWebView(originalRequest: Request): Request {
-                it.name.startsWith("__cf") || it.name.startsWith("cf_")
+        val url = originalRequest.url.toString()
-            }.map {
+
-                // Convert cookies -> OkHttp format
+        logger.debug { "resolveWithWebView($url)" }
-                Cookie.Builder()
+
-                    .domain(it.domain.removePrefix("."))
+        val cookies = PythonInterpreter.create().use { py ->
-                    .expiresAt(it.expires?.time ?: Long.MAX_VALUE)
+            try {
-                    .name(it.name)
+                py.exec("import undetected_chromedriver as uc")
-                    .path(it.path)
+
-                    .value(it.value).apply {
+                py.exec("options = uc.ChromeOptions()")
-                        if (it.isHttpOnly) httpOnly()
+
-                        if (it.isSecure) secure()
+                if (serverConfig.socksProxyEnabled) {
-                    }.build()
+                    val proxy = "socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}"
+                    py.exec("options.add_argument('--proxy-server=$proxy')")
+                }
+//                py.exec("driver = uc.Chrome(options=options)")
+                py.exec("driver = uc.Chrome(options=options, driver_executable_path='${py.chromedriverPath.replace("\\","\\\\")}', version_main=111)")
+
+                // TODO: handle custom userAgent
+//                val userAgent = originalRequest.header("User-Agent")
+
+//                if (userAgent != null) {
+//                    browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
+//                        browserContext.newPage().use { getCookies(it, url) }
+//                    }
+//                } else {
+//                    browser.newPage().use { getCookies(it, url) }
+//                }
+                py.exec("driver.get('$url')")
+
+                getCookies(py)
+            } finally {
+                py.exec("driver.quit()")
             }
         }
 
         // Copy cookies to cookie store
-        convertedCookies.forEach {
+        cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
             network.cookies.addAll(
-                HttpUrl.Builder()
+                url = HttpUrl.Builder()
                     .scheme("http")
-                    .host(it.domain)
+                    .host(domain)
                     .build(),
-                listOf(it)
+                cookies = cookies
             )
         }
         // Merge new and existing cookies for this request
         // Find the cookies that we need to merge into this request
-        val convertedForThisRequest = convertedCookies.filter {
+        val convertedForThisRequest = cookies.filter {
-            it.matches(response.request.url)
+            it.matches(originalRequest.url)
         }
         // Extract cookies from current request
         val existingCookies = Cookie.parseAll(
-            response.request.url,
+            originalRequest.url,
-            response.request.headers
+            originalRequest.headers
         )
         // Filter out existing values of cookies that we are about to merge in
         val filteredExisting = existingCookies.filter { existing ->
             convertedForThisRequest.none { converted -> converted.name == existing.name }
         }
+        logger.trace { "Existing cookies" }
+        logger.trace { existingCookies.joinToString("; ") }
         val newCookies = filteredExisting + convertedForThisRequest
-        return response.request.newBuilder()
+        logger.trace { "New cookies" }
-            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+        logger.trace { newCookies.joinToString("; ") }
+        return originalRequest.newBuilder()
+            .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" })
             .build()
     }
 
-    companion object {
+    fun getWebViewUserAgent(): String {
-        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+        return try {
-        private val COOKIE_NAMES = listOf("cf_clearance")
+            if (!serverConfig.webviewEnabled) {
+                throw CloudflareBypassException("Webview is disabled, enable it in server config")
+            }
+
+            PythonInterpreter.create().use { py ->
+                py.exec("import undetected_chromedriver as uc")
+
+                py.exec("options = uc.ChromeOptions()")
+                py.exec("options.add_argument('--headless')")
+                py.exec("options.add_argument('--disable-gpu')")
+//                py.exec("driver = uc.Chrome(options=options)")
+                py.exec("driver = uc.Chrome(options=options, driver_executable_path='${py.chromedriverPath.replace("\\","\\\\")}', version_main=111)")
+
+                py.exec("userAgent = driver.execute_script('return navigator.userAgent')")
+                val userAgent = py.getValue("userAgent")
+                py.exec("driver.quit()")
+
+                userAgent
+            }
+        } catch (e: Exception) {
+            // Webview might fail on headless environments like docker
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
+        }
+    }
+
+    @Serializable
+    data class PythonSeleniumCookie(
+        val domain: String,
+        val expiry: Long?,
+        val httpOnly: Boolean,
+        val name: String,
+        val path: String,
+        val sameSite: String,
+        val secure: Boolean,
+        val value: String
+    )
+
+    private val json by DI.global.instance<Json>()
+    private fun getCookies(py: PythonInterpreter): List<Cookie> {
+        val challengeResolved = waitForChallengeResolve(py)
+
+        return if (challengeResolved) {
+            py.exec("import json")
+            py.exec("cookies = json.dumps(driver.get_cookies())")
+            val cookiesJson = py.getValue("cookies")
+            val cookies = json.decodeFromString<List<PythonSeleniumCookie>>(cookiesJson)
+
+            logger.debug {
+                py.exec("userAgent = driver.execute_script('return navigator.userAgent')")
+                val userAgent = py.getValue("userAgent")
+                "Webview User-Agent is $userAgent"
+            }
+
+            // Convert Webview cookies to OkHttp cookies
+            cookies.map {
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expiry?.times(1000) ?: Long.MAX_VALUE)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.httpOnly) httpOnly()
+                        if (it.secure) secure()
+                    }.build()
+            }
+        } else {
+            throw CloudflareBypassException("Cloudflare challenge failed to resolve")
+        }
+    }
+
+    private fun waitForChallengeResolve(py: PythonInterpreter): Boolean {
+        // sometimes the user has to solve the captcha challenge manually and multiple times, potentially wait a long time
+        val timeoutSeconds = 120
+        repeat(timeoutSeconds) {
+            TimeUnit.SECONDS.sleep(1)
+            val success = try {
+                py.exec("r = driver.execute_script('return document.querySelector(\"#challenge-form\") == null')")
+                py.getValue("r").lowercase().toBoolean()
+            } catch (e: Exception) {
+                logger.debug(e) { "query Error" }
+                false
+            }
+            if (success) return true
+        }
+        return false
+    }
+}
+private class CloudflareBypassException(message: String?) : Exception(message)
+
+class PythonInterpreter
+private constructor(private val process: Process, val chromedriverPath: String) : Closeable {
+    private val stdin = process.outputStream
+    private val stdout = process.inputStream
+    private val stderr = process.errorStream
+
+    private val stdinWriter = PrintWriter(stdin)
+    private val stdoutReader = BufferedReader(InputStreamReader(stdout))
+    private val stderrReader = BufferedReader(InputStreamReader(stderr))
+
+    private fun rawExec(command: String) {
+        stdinWriter.println(command)
+        stdinWriter.flush()
+    }
+
+    val BUFF_SIZE = 102400
+    fun exec(command: String) {
+        logger.debug { "Python Command: $command" }
+        rawExec(command)
+        makeSureExecDone()
+    }
+
+    private val commandOutputs = mutableListOf<String>()
+
+    fun makeSureExecDone() {
+        val makeSureString = "PYTHON_IS_READY"
+
+        rawExec("print('$makeSureString')")
+        var line: String?
+        do {
+            line = stdoutReader.readLine()
+            if (line != makeSureString) {
+                commandOutputs.add(line)
+            }
+        } while (line != makeSureString)
+
+        val pyError = buildString {
+            while (stderrReader.ready())
+                append(stderr.read().toChar())
+        }
+        if (pyError.isNotEmpty()) {
+            println("Python STDERR: $pyError")
+        }
+    }
+
+    fun getValue(variableName: String): String {
+        exec("print($variableName)")
+        return commandOutputs.last()
+    }
+
+    private fun flushStdoutReader() {
+        var line: String?
+        while (stdoutReader.ready()) {
+            val line = stdoutReader.readLine()
+        }
+    }
+
+    fun destroy() {
+        stdinWriter.close()
+        stdoutReader.close()
+        stderr.close()
+        process.destroy()
+    }
+    override fun close() {
+        destroy()
+    }
+    companion object {
+        private val logger = KotlinLogging.logger {}
+
+        fun create(pythonPath: String, workingDir: String, pythonStartupFile: String, chromedriverPath: String): PythonInterpreter {
+            val processBuilder = ProcessBuilder()
+                .command(pythonPath, "-i", "-q")
+            processBuilder.directory(File(workingDir))
+
+            val environment = processBuilder.environment()
+            environment["PYTHONSTARTUP"] = pythonStartupFile
+
+            val process = processBuilder.start()
+
+            return PythonInterpreter(process, chromedriverPath)
+        }
+
+        fun create(): PythonInterpreter {
+            val uc = Paths.get(serverConfig.undetectedChromePath).toAbsolutePath().toString()
+            logger.debug { "absolute path to undetected-chromedriver: $uc" }
+
+            val (pythonPath, chromedriverPath) = if (System.getProperty("os.name").startsWith("Windows")) {
+                arrayOf(
+                    "$uc\\venv\\Scripts\\python.exe",
+                    "$uc\\chromedriver.exe"
+                )
+            } else {
+                arrayOf(
+                    "$uc/venv/bin/python",
+                    "$uc/chromedriver"
+                )
+            }
+
+            return create(
+                pythonPath,
+                uc,
+                "$uc/console.py",
+                chromedriverPath
+            )
+        }
     }
 }

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt

@@ -20,6 +20,8 @@ interface SManga : Serializable {
 
     var thumbnail_url: String?
 
+    var update_strategy: UpdateStrategy
+
     var initialized: Boolean
 
     fun copyFrom(other: SManga) {

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt

@@ -18,5 +18,7 @@ class SMangaImpl : SManga {
 
     override var thumbnail_url: String? = null
 
+    override var update_strategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE
+
     override var initialized: Boolean = false
 }

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt

@@ -0,0 +1,6 @@
+package eu.kanade.tachiyomi.source.model
+
+enum class UpdateStrategy {
+    ALWAYS_UPDATE,
+    ONLY_FETCH_ONCE
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.source.online
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.getWebViewUserAgent
 import eu.kanade.tachiyomi.network.newCallWithProgress
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -356,6 +357,28 @@ abstract class HttpSource : CatalogueSource {
         }
     }
 
+    /**
+     * Returns the url of the provided manga
+     *
+     * @since extensions-lib 1.4
+     * @param manga the manga
+     * @return url of the manga
+     */
+    open fun getMangaUrl(manga: SManga): String {
+        return mangaDetailsRequest(manga).url.toString()
+    }
+
+    /**
+     * Returns the url of the provided chapter
+     *
+     * @since extensions-lib 1.4
+     * @param chapter the chapter
+     * @return url of the chapter
+     */
+    open fun getChapterUrl(chapter: SChapter): String {
+        return pageListRequest(chapter).url.toString()
+    }
+
     /**
      * Called before inserting a new chapter into database. Use it if you need to override chapter
      * fields, like the title or the chapter number. Do not change anything to [manga].
@@ -363,8 +386,7 @@ abstract class HttpSource : CatalogueSource {
      * @param chapter the chapter to be added.
      * @param manga the manga of the chapter.
      */
-    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {
+    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {}
-    }
 
     /**
      * Returns the list of filters for the source.
@@ -372,6 +394,6 @@ abstract class HttpSource : CatalogueSource {
     override fun getFilterList() = FilterList()
 
     companion object {
-        const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63"
+        val DEFAULT_USER_AGENT by lazy { getWebViewUserAgent() }
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/CategoryController.kt

@@ -61,14 +61,15 @@ object CategoryController {
         pathParam<Int>("categoryId"),
         formParam<String?>("name"),
         formParam<Boolean?>("default"),
+        formParam<Int?>("includeInUpdate"),
         documentWith = {
             withOperation {
                 summary("Category modify")
                 description("Modify a category")
             }
         },
-        behaviorOf = { ctx, categoryId, name, isDefault ->
+        behaviorOf = { ctx, categoryId, name, isDefault, includeInUpdate ->
-            Category.updateCategory(categoryId, name, isDefault)
+            Category.updateCategory(categoryId, name, isDefault, includeInUpdate)
             ctx.status(200)
         },
         withResults = {

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt

@@ -15,7 +15,6 @@ import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import suwayomi.tachidesk.server.JavalinSetup.future
 import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
-import suwayomi.tachidesk.server.util.queryParam
 import suwayomi.tachidesk.server.util.withOperation
 
 object ExtensionController {
@@ -141,16 +140,15 @@ object ExtensionController {
     /** icon for extension named `apkName` */
     val icon = handler(
         pathParam<String>("apkName"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Extension icon")
                 description("Icon for extension named `apkName`")
             }
         },
-        behaviorOf = { ctx, apkName, useCache ->
+        behaviorOf = { ctx, apkName ->
             ctx.future(
-                future { Extension.getExtensionIcon(apkName, useCache) }
+                future { Extension.getExtensionIcon(apkName) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         it.first

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/MangaController.kt

@@ -81,16 +81,15 @@ object MangaController {
     /** manga thumbnail */
     val thumbnail = handler(
         pathParam<Int>("mangaId"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Get a manga thumbnail")
                 description("Get a manga thumbnail from the source or the cache.")
             }
         },
-        behaviorOf = { ctx, mangaId, useCache ->
+        behaviorOf = { ctx, mangaId ->
             ctx.future(
-                future { Manga.getMangaThumbnail(mangaId, useCache) }
+                future { Manga.getMangaThumbnail(mangaId) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         val httpCacheSeconds = 1.days.inWholeSeconds
@@ -375,16 +374,15 @@ object MangaController {
         pathParam<Int>("mangaId"),
         pathParam<Int>("chapterIndex"),
         pathParam<Int>("index"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Get a chapter page")
                 description("Get a chapter page for a given index. Cache use can be disabled so it only retrieves it directly from the source.")
             }
         },
-        behaviorOf = { ctx, mangaId, chapterIndex, index, useCache ->
+        behaviorOf = { ctx, mangaId, chapterIndex, index ->
             ctx.future(
-                future { Page.getPageImage(mangaId, chapterIndex, index, useCache) }
+                future { Page.getPageImage(mangaId, chapterIndex, index) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         val httpCacheSeconds = 1.days.inWholeSeconds

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt

@@ -1,5 +1,13 @@
 package suwayomi.tachidesk.manga.controller
 
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import io.javalin.http.HttpCode
 import io.javalin.websocket.WsConfig
 import mu.KotlinLogging
@@ -12,23 +20,13 @@ import suwayomi.tachidesk.manga.impl.Chapter
 import suwayomi.tachidesk.manga.impl.update.IUpdater
 import suwayomi.tachidesk.manga.impl.update.UpdateStatus
 import suwayomi.tachidesk.manga.impl.update.UpdaterSocket
-import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
+import suwayomi.tachidesk.manga.model.dataclass.*
-import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass
-import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
-import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
 import suwayomi.tachidesk.server.JavalinSetup.future
 import suwayomi.tachidesk.server.util.formParam
 import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
 import suwayomi.tachidesk.server.util.withOperation
 
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
 object UpdateController {
     private val logger = KotlinLogging.logger { }
 
@@ -92,13 +90,33 @@ object UpdateController {
         if (clear) {
             updater.reset()
         }
-        categories
+
+        val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
+        val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.EXCLUDE].orEmpty()
+        val includedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.INCLUDE].orEmpty()
+        val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.UNSET].orEmpty()
+        val categoriesToUpdate = includedCategories.ifEmpty { unsetCategories }
+
+        logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
+
+        val categoriesToUpdateMangas = categoriesToUpdate
             .flatMap { CategoryManga.getCategoryMangaList(it.id) }
             .distinctBy { it.id }
-            .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
+        val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
-            .forEach { manga ->
+        val mangasToUpdate = categoriesToUpdateMangas
-                updater.addMangaToQueue(manga)
+            .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
-            }
+            .filter { !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
+
+        // In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
+        if (mangasToUpdate.isEmpty()) {
+            UpdaterSocket.notifyAllClients(UpdateStatus())
+            return
+        }
+
+        updater.addMangasToQueue(
+            mangasToUpdate
+                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
+        )
     }
 
     fun categoryUpdateWS(ws: WsConfig) {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt

@@ -18,8 +18,8 @@ import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.CategoryManga.removeMangaFromCategory
-import suwayomi.tachidesk.manga.impl.util.lang.isNotEmpty
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
+import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
 import suwayomi.tachidesk.manga.model.table.CategoryMetaTable
 import suwayomi.tachidesk.manga.model.table.CategoryTable
@@ -50,11 +50,12 @@ object Category {
         }
     }
 
-    fun updateCategory(categoryId: Int, name: String?, isDefault: Boolean?) {
+    fun updateCategory(categoryId: Int, name: String?, isDefault: Boolean?, includeInUpdate: Int?) {
         transaction {
             CategoryTable.update({ CategoryTable.id eq categoryId }) {
                 if (name != null && !name.equals(DEFAULT_CATEGORY_NAME, ignoreCase = true)) it[CategoryTable.name] = name
                 if (isDefault != null) it[CategoryTable.isDefault] = isDefault
+                if (includeInUpdate != null) it[CategoryTable.includeInUpdate] = includeInUpdate
             }
         }
     }
@@ -98,12 +99,14 @@ object Category {
 
     const val DEFAULT_CATEGORY_ID = 0
     const val DEFAULT_CATEGORY_NAME = "Default"
-    private fun addDefaultIfNecessary(categories: List<CategoryDataClass>): List<CategoryDataClass> =
+    private fun addDefaultIfNecessary(categories: List<CategoryDataClass>): List<CategoryDataClass> {
-        if (MangaTable.select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }.isNotEmpty()) {
+        val defaultCategorySize = MangaTable.select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }.count().toInt()
-            listOf(CategoryDataClass(DEFAULT_CATEGORY_ID, 0, DEFAULT_CATEGORY_NAME, true)) + categories
+        return if (defaultCategorySize > 0) {
+            listOf(CategoryDataClass(DEFAULT_CATEGORY_ID, 0, DEFAULT_CATEGORY_NAME, true, defaultCategorySize, IncludeInUpdate.UNSET)) + categories
         } else {
             categories
         }
+    }
 
     fun getCategoryList(): List<CategoryDataClass> {
         return transaction {
@@ -123,6 +126,14 @@ object Category {
         }
     }
 
+    fun getCategorySize(categoryId: Int): Int {
+        return transaction {
+            CategoryMangaTable.select {
+                CategoryMangaTable.category eq categoryId
+            }.count().toInt()
+        }
+    }
+
     fun getCategoryMetaMap(categoryId: Int): Map<String, String> {
         return transaction {
             CategoryMetaTable.select { CategoryMetaTable.ref eq categoryId }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt

@@ -10,10 +10,13 @@ package suwayomi.tachidesk.manga.impl
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
+import org.jetbrains.exposed.sql.alias
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.count
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.leftJoin
+import org.jetbrains.exposed.sql.max
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
@@ -61,46 +64,45 @@ object CategoryManga {
      * list of mangas that belong to a category
      */
     fun getCategoryMangaList(categoryId: Int): List<MangaDataClass> {
-        val unreadExpression = wrapAsExpression<Long>(
+        // Select the required columns from the MangaTable and add the aggregate functions to compute unread, download, and chapter counts
-            ChapterTable
+        val unreadCount = wrapAsExpression<Long>(
-                .slice(ChapterTable.id.count())
+            ChapterTable.slice(ChapterTable.id.count()).select((ChapterTable.isRead eq false) and (ChapterTable.manga eq MangaTable.id))
-                .select { (MangaTable.id eq ChapterTable.manga) and (ChapterTable.isRead eq false) }
         )
-        val downloadExpression = wrapAsExpression<Long>(
+        val downloadedCount = wrapAsExpression<Long>(
-            ChapterTable
+            ChapterTable.slice(ChapterTable.id.count()).select((ChapterTable.isDownloaded eq true) and (ChapterTable.manga eq MangaTable.id))
-                .slice(ChapterTable.id.count())
-                .select { (MangaTable.id eq ChapterTable.manga) and (ChapterTable.isDownloaded eq true) }
-        )
-        val chapterCountExpression = wrapAsExpression<Long>(
-            ChapterTable
-                .slice(ChapterTable.id.count())
-                .select { (MangaTable.id eq ChapterTable.manga) }
         )
 
-        val selectedColumns = MangaTable.columns + unreadExpression + downloadExpression + chapterCountExpression
+        val chapterCount = ChapterTable.id.count().alias("chapter_count")
+        val lastReadAt = ChapterTable.lastReadAt.max().alias("last_read_at")
+        val selectedColumns = MangaTable.columns + unreadCount + downloadedCount + chapterCount + lastReadAt
 
         val transform: (ResultRow) -> MangaDataClass = {
+            // Map the data from the result row to the MangaDataClass
             val dataClass = MangaTable.toDataClass(it)
-            dataClass.unreadCount = it[unreadExpression]
+            dataClass.lastReadAt = it[lastReadAt]
-            dataClass.downloadCount = it[downloadExpression]
+            dataClass.unreadCount = it[unreadCount]
-            dataClass.chapterCount = it[chapterCountExpression]
+            dataClass.downloadCount = it[downloadedCount]
+            dataClass.chapterCount = it[chapterCount]
             dataClass
         }
 
-        if (categoryId == DEFAULT_CATEGORY_ID) {
-            return transaction {
-                MangaTable
-                    .slice(selectedColumns)
-                    .select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }
-                    .map(transform)
-            }
-        }
-
         return transaction {
-            CategoryMangaTable.innerJoin(MangaTable)
+            // Fetch data from the MangaTable and join with the CategoryMangaTable, if a category is specified
-                .slice(selectedColumns)
+            val query = if (categoryId == DEFAULT_CATEGORY_ID) {
-                .select { (MangaTable.inLibrary eq true) and (CategoryMangaTable.category eq categoryId) }
+                MangaTable
-                .map(transform)
+                    .leftJoin(ChapterTable, { MangaTable.id }, { ChapterTable.manga })
+                    .slice(columns = selectedColumns)
+                    .select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }
+            } else {
+                MangaTable
+                    .innerJoin(CategoryMangaTable)
+                    .leftJoin(ChapterTable, { MangaTable.id }, { ChapterTable.manga })
+                    .slice(columns = selectedColumns)
+                    .select { (MangaTable.inLibrary eq true) and (CategoryMangaTable.category eq categoryId) }
+            }
+
+            // Join with the ChapterTable to fetch the last read chapter for each manga
+            query.groupBy(*MangaTable.columns.toTypedArray()).map(transform)
         }
     }
 
@@ -114,4 +116,20 @@ object CategoryManga {
             }
         }
     }
+
+    fun getMangasCategories(mangaIDs: List<Int>): Map<Int, List<CategoryDataClass>> {
+        return buildMap {
+            transaction {
+                CategoryMangaTable.innerJoin(CategoryTable)
+                    .select { CategoryMangaTable.manga inList mangaIDs }
+                    .groupBy { it[CategoryMangaTable.manga] }
+                    .forEach {
+                        val mangaId = it.key.value
+                        val categories = it.value
+
+                        set(mangaId, categories.map { category -> CategoryTable.toDataClass(category) })
+                    }
+            }
+        }
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -12,13 +12,17 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
 import kotlinx.serialization.Serializable
 import org.jetbrains.exposed.dao.id.EntityID
-import org.jetbrains.exposed.sql.*
+import org.jetbrains.exposed.sql.Op
+import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SortOrder.ASC
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
-import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
+import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.deleteWhere
+import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
@@ -27,10 +31,10 @@ import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
 import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom
 import suwayomi.tachidesk.manga.model.table.ChapterMetaTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.ChapterTable.scanlator
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
-import java.io.File
 import java.time.Instant
 
 object Chapter {
@@ -85,6 +89,10 @@ object Chapter {
                         it[sourceOrder] = index + 1
                         it[fetchedAt] = now++
                         it[ChapterTable.manga] = mangaId
+
+                        it[realUrl] = runCatching {
+                            (source as? HttpSource)?.getChapterUrl(fetchedChapter)
+                        }.getOrNull()
                     }
                 } else {
                     ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) {
@@ -95,6 +103,10 @@ object Chapter {
 
                         it[sourceOrder] = index + 1
                         it[ChapterTable.manga] = mangaId
+
+                        it[realUrl] = runCatching {
+                            (source as? HttpSource)?.getChapterUrl(fetchedChapter)
+                        }.getOrNull()
                     }
                 }
             }
@@ -138,26 +150,27 @@ object Chapter {
             val dbChapter = dbChapterMap.getValue(it.url)
 
             ChapterDataClass(
-                dbChapter[ChapterTable.id].value,
+                id = dbChapter[ChapterTable.id].value,
-                it.url,
+                url = it.url,
-                it.name,
+                name = it.name,
-                it.date_upload,
+                uploadDate = it.date_upload,
-                it.chapter_number,
+                chapterNumber = it.chapter_number,
-                it.scanlator,
+                scanlator = it.scanlator,
-                mangaId,
+                mangaId = mangaId,
 
-                dbChapter[ChapterTable.isRead],
+                read = dbChapter[ChapterTable.isRead],
-                dbChapter[ChapterTable.isBookmarked],
+                bookmarked = dbChapter[ChapterTable.isBookmarked],
-                dbChapter[ChapterTable.lastPageRead],
+                lastPageRead = dbChapter[ChapterTable.lastPageRead],
-                dbChapter[ChapterTable.lastReadAt],
+                lastReadAt = dbChapter[ChapterTable.lastReadAt],
 
-                chapterCount - index,
+                index = chapterCount - index,
-                dbChapter[ChapterTable.fetchedAt],
+                fetchedAt = dbChapter[ChapterTable.fetchedAt],
-                dbChapter[ChapterTable.isDownloaded],
+                realUrl = dbChapter[ChapterTable.realUrl],
+                downloaded = dbChapter[ChapterTable.isDownloaded],
 
-                dbChapter[ChapterTable.pageCount],
+                pageCount = dbChapter[ChapterTable.pageCount],
 
-                chapterList.size,
+                chapterCount = chapterList.size,
                 meta = chapterMetas.getValue(dbChapter[ChapterTable.id])
             )
         }
@@ -312,9 +325,7 @@ object Chapter {
                 ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder eq chapterIndex) }
                     .first()[ChapterTable.id].value
 
-            val chapterDir = getChapterDir(mangaId, chapterId)
+            ChapterDownloadHelper.delete(mangaId, chapterId)
-
-            File(chapterDir).deleteRecursively()
 
             ChapterTable.update({ (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder eq chapterIndex) }) {
                 it[isDownloaded] = false
@@ -332,8 +343,7 @@ object Chapter {
                     .forEach { row ->
                         val chapterMangaId = row[ChapterTable.manga].value
                         val chapterId = row[ChapterTable.id].value
-                        val chapterDir = getChapterDir(chapterMangaId, chapterId)
+                        ChapterDownloadHelper.delete(chapterMangaId, chapterId)
-                        File(chapterDir).deleteRecursively()
                     }
 
                 ChapterTable.update({ ChapterTable.id inList chapterIds }) {
@@ -346,8 +356,8 @@ object Chapter {
                     .select { (ChapterTable.sourceOrder inList input.chapterIndexes) and (ChapterTable.manga eq mangaId) }
                     .map { row ->
                         val chapterId = row[ChapterTable.id].value
-                        val chapterDir = getChapterDir(mangaId, chapterId)
+                        ChapterDownloadHelper.delete(mangaId, chapterId)
-                        File(chapterDir).deleteRecursively()
+
                         chapterId
                     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt

@@ -0,0 +1,41 @@
+package suwayomi.tachidesk.manga.impl
+
+import kotlinx.coroutines.CoroutineScope
+import suwayomi.tachidesk.manga.impl.download.ArchiveProvider
+import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider
+import suwayomi.tachidesk.manga.impl.download.FolderProvider
+import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
+import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
+import suwayomi.tachidesk.server.serverConfig
+import java.io.File
+import java.io.InputStream
+
+object ChapterDownloadHelper {
+    fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair<InputStream, String> {
+        return provider(mangaId, chapterId).getImage(index)
+    }
+
+    fun delete(mangaId: Int, chapterId: Int): Boolean {
+        return provider(mangaId, chapterId).delete()
+    }
+
+    suspend fun download(
+        mangaId: Int,
+        chapterId: Int,
+        download: DownloadChapter,
+        scope: CoroutineScope,
+        step: suspend (DownloadChapter?, Boolean) -> Unit
+    ): Boolean {
+        return provider(mangaId, chapterId).download(download, scope, step)
+    }
+
+    // return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available
+    private fun provider(mangaId: Int, chapterId: Int): DownloadedFilesProvider {
+        val chapterFolder = File(getChapterDownloadPath(mangaId, chapterId))
+        val cbzFile = File(getChapterCbzPath(mangaId, chapterId))
+        if (cbzFile.exists()) return ArchiveProvider(mangaId, chapterId)
+        if (!chapterFolder.exists() && serverConfig.downloadAsCbz) return ArchiveProvider(mangaId, chapterId)
+        return FolderProvider(mangaId, chapterId)
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import eu.kanade.tachiyomi.source.online.HttpSource
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
@@ -30,7 +31,7 @@ import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogue
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.impl.util.source.StubSource
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
@@ -87,45 +88,49 @@ object Manga {
                     it[MangaTable.description] = truncate(sManga.description, 4096)
                     it[MangaTable.genre] = sManga.genre
                     it[MangaTable.status] = sManga.status
-                    if (sManga.thumbnail_url != null && sManga.thumbnail_url.orEmpty().isNotEmpty()) {
+                    if (!sManga.thumbnail_url.isNullOrEmpty() && sManga.thumbnail_url != mangaEntry[MangaTable.thumbnail_url]) {
                         it[MangaTable.thumbnail_url] = sManga.thumbnail_url
+                        it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond
+                        clearMangaThumbnailCache(mangaId)
                     }
 
                     it[MangaTable.realUrl] = runCatching {
-                        (source as? HttpSource)?.mangaDetailsRequest(sManga)?.url?.toString()
+                        (source as? HttpSource)?.getMangaUrl(sManga)
                     }.getOrNull()
 
                     it[MangaTable.lastFetchedAt] = Instant.now().epochSecond
+
+                    it[MangaTable.updateStrategy] = sManga.update_strategy.name
                 }
             }
 
-            clearMangaThumbnail(mangaId)
-
             mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
 
             MangaDataClass(
-                mangaId,
+                id = mangaId,
-                mangaEntry[MangaTable.sourceReference].toString(),
+                sourceId = mangaEntry[MangaTable.sourceReference].toString(),
 
-                mangaEntry[MangaTable.url],
+                url = mangaEntry[MangaTable.url],
-                mangaEntry[MangaTable.title],
+                title = mangaEntry[MangaTable.title],
-                proxyThumbnailUrl(mangaId),
+                thumbnailUrl = proxyThumbnailUrl(mangaId),
+                thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
-                true,
+                initialized = true,
 
-                sManga.artist,
+                artist = sManga.artist,
-                sManga.author,
+                author = sManga.author,
-                sManga.description,
+                description = sManga.description,
-                sManga.genre.toGenreList(),
+                genre = sManga.genre.toGenreList(),
-                MangaStatus.valueOf(sManga.status).name,
+                status = MangaStatus.valueOf(sManga.status).name,
-                mangaEntry[MangaTable.inLibrary],
+                inLibrary = mangaEntry[MangaTable.inLibrary],
-                mangaEntry[MangaTable.inLibraryAt],
+                inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
-                getSource(mangaEntry[MangaTable.sourceReference]),
+                source = getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaId),
+                meta = getMangaMetaMap(mangaId),
-                mangaEntry[MangaTable.realUrl],
+                realUrl = mangaEntry[MangaTable.realUrl],
-                mangaEntry[MangaTable.lastFetchedAt],
+                lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
-                mangaEntry[MangaTable.chaptersLastFetchedAt],
+                chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
-                true
+                updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
+                freshData = true
             )
         }
     }
@@ -165,28 +170,30 @@ object Manga {
     }
 
     private fun getMangaDataClass(mangaId: Int, mangaEntry: ResultRow) = MangaDataClass(
-        mangaId,
+        id = mangaId,
-        mangaEntry[MangaTable.sourceReference].toString(),
+        sourceId = mangaEntry[MangaTable.sourceReference].toString(),
 
-        mangaEntry[MangaTable.url],
+        url = mangaEntry[MangaTable.url],
-        mangaEntry[MangaTable.title],
+        title = mangaEntry[MangaTable.title],
-        proxyThumbnailUrl(mangaId),
+        thumbnailUrl = proxyThumbnailUrl(mangaId),
+        thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
-        true,
+        initialized = true,
 
-        mangaEntry[MangaTable.artist],
+        artist = mangaEntry[MangaTable.artist],
-        mangaEntry[MangaTable.author],
+        author = mangaEntry[MangaTable.author],
-        mangaEntry[MangaTable.description],
+        description = mangaEntry[MangaTable.description],
-        mangaEntry[MangaTable.genre].toGenreList(),
+        genre = mangaEntry[MangaTable.genre].toGenreList(),
-        MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+        status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-        mangaEntry[MangaTable.inLibrary],
+        inLibrary = mangaEntry[MangaTable.inLibrary],
-        mangaEntry[MangaTable.inLibraryAt],
+        inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
-        getSource(mangaEntry[MangaTable.sourceReference]),
+        source = getSource(mangaEntry[MangaTable.sourceReference]),
-        getMangaMetaMap(mangaId),
+        meta = getMangaMetaMap(mangaId),
-        mangaEntry[MangaTable.realUrl],
+        realUrl = mangaEntry[MangaTable.realUrl],
-        mangaEntry[MangaTable.lastFetchedAt],
+        lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
-        mangaEntry[MangaTable.chaptersLastFetchedAt],
+        chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
-        false
+        updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
+        freshData = false
     )
 
     fun getMangaMetaMap(mangaId: Int): Map<String, String> {
@@ -218,15 +225,15 @@ object Manga {
 
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
     private val network: NetworkHelper by injectLazy()
-    suspend fun getMangaThumbnail(mangaId: Int, useCache: Boolean): Pair<InputStream, String> {
+    suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
-        val saveDir = applicationDirs.thumbnailsRoot
+        val cacheSaveDir = applicationDirs.thumbnailsRoot
         val fileName = mangaId.toString()
 
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val sourceId = mangaEntry[MangaTable.sourceReference]
 
         return when (val source = getCatalogueSourceOrStub(sourceId)) {
-            is HttpSource -> getImageResponse(saveDir, fileName, useCache) {
+            is HttpSource -> getCachedImageResponse(cacheSaveDir, fileName) {
                 val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
                     ?: if (!mangaEntry[MangaTable.initialized]) {
                         // initialize then try again
@@ -258,7 +265,7 @@ object Manga {
                 imageFile.inputStream() to contentType
             }
 
-            is StubSource -> getImageResponse(saveDir, fileName, useCache) {
+            is StubSource -> getCachedImageResponse(cacheSaveDir, fileName) {
                 val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
                     ?: throw NullPointerException("No thumbnail found")
                 network.client.newCall(
@@ -270,7 +277,7 @@ object Manga {
         }
     }
 
-    private fun clearMangaThumbnail(mangaId: Int) {
+    private fun clearMangaThumbnailCache(mangaId: Int) {
         val saveDir = applicationDirs.thumbnailsRoot
         val fileName = mangaId.toString()
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt

@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
@@ -61,6 +62,7 @@ object MangaList {
                         it[genre] = manga.genre
                         it[status] = manga.status
                         it[thumbnail_url] = manga.thumbnail_url
+                        it[updateStrategy] = manga.update_strategy.name
 
                         it[sourceReference] = sourceId
                     }.value
@@ -70,51 +72,55 @@ object MangaList {
                     }.first()
 
                     MangaDataClass(
-                        mangaId,
+                        id = mangaId,
-                        sourceId.toString(),
+                        sourceId = sourceId.toString(),
 
-                        manga.url,
+                        url = manga.url,
-                        manga.title,
+                        title = manga.title,
-                        proxyThumbnailUrl(mangaId),
+                        thumbnailUrl = proxyThumbnailUrl(mangaId),
+                        thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
-                        manga.initialized,
+                        initialized = manga.initialized,
 
-                        manga.artist,
+                        artist = manga.artist,
-                        manga.author,
+                        author = manga.author,
-                        manga.description,
+                        description = manga.description,
-                        manga.genre.toGenreList(),
+                        genre = manga.genre.toGenreList(),
-                        MangaStatus.valueOf(manga.status).name,
+                        status = MangaStatus.valueOf(manga.status).name,
-                        false, // It's a new manga entry
+                        inLibrary = false, // It's a new manga entry
-                        0,
+                        inLibraryAt = 0,
                         meta = getMangaMetaMap(mangaId),
                         realUrl = mangaEntry[MangaTable.realUrl],
                         lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
                         chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
+                        updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
                         freshData = true
                     )
                 } else {
                     val mangaId = mangaEntry[MangaTable.id].value
                     MangaDataClass(
-                        mangaId,
+                        id = mangaId,
-                        sourceId.toString(),
+                        sourceId = sourceId.toString(),
 
-                        manga.url,
+                        url = manga.url,
-                        manga.title,
+                        title = manga.title,
-                        proxyThumbnailUrl(mangaId),
+                        thumbnailUrl = proxyThumbnailUrl(mangaId),
+                        thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
-                        true,
+                        initialized = true,
 
-                        mangaEntry[MangaTable.artist],
+                        artist = mangaEntry[MangaTable.artist],
-                        mangaEntry[MangaTable.author],
+                        author = mangaEntry[MangaTable.author],
-                        mangaEntry[MangaTable.description],
+                        description = mangaEntry[MangaTable.description],
-                        mangaEntry[MangaTable.genre].toGenreList(),
+                        genre = mangaEntry[MangaTable.genre].toGenreList(),
-                        MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                        status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-                        mangaEntry[MangaTable.inLibrary],
+                        inLibrary = mangaEntry[MangaTable.inLibrary],
-                        mangaEntry[MangaTable.inLibraryAt],
+                        inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
                         meta = getMangaMetaMap(mangaId),
                         realUrl = mangaEntry[MangaTable.realUrl],
                         lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
                         chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
+                        updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
                         freshData = false
                     )
                 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt

@@ -11,14 +11,15 @@ import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.online.HttpSource
 import kotlinx.coroutines.flow.StateFlow
+import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
+import suwayomi.tachidesk.manga.impl.util.getChapterCachePath
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
@@ -38,7 +39,7 @@ object Page {
         return page.imageUrl!!
     }
 
-    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, useCache: Boolean = true, progressFlow: ((StateFlow<Int>) -> Unit)? = null): Pair<InputStream, String> {
+    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, progressFlow: ((StateFlow<Int>) -> Unit)? = null): Pair<InputStream, String> {
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
         val chapterEntry = transaction {
@@ -49,8 +50,11 @@ object Page {
         val chapterId = chapterEntry[ChapterTable.id].value
 
         val pageEntry =
-            transaction { PageTable.select { (PageTable.chapter eq chapterId) and (PageTable.index eq index) }.first() }
+            transaction {
-
+                PageTable.select { (PageTable.chapter eq chapterId) }
+                    .orderBy(PageTable.index to SortOrder.ASC)
+                    .limit(1, index.toLong()).first()
+            }
         val tachiyomiPage = Page(
             pageEntry[PageTable.index],
             pageEntry[PageTable.url],
@@ -82,11 +86,16 @@ object Page {
             }
         }
 
-        val chapterDir = getChapterDir(mangaId, chapterId)
-        File(chapterDir).mkdirs()
         val fileName = getPageName(index)
 
-        return getImageResponse(chapterDir, fileName, useCache) {
+        if (chapterEntry[ChapterTable.isDownloaded]) {
+            return ChapterDownloadHelper.getImage(mangaId, chapterId, index)
+        }
+
+        val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
+
+        // Note: don't care about invalidating cache because OS cache is not permanent
+        return getCachedImageResponse(cacheSaveDir, fileName) {
             source.fetchImage(tachiyomiPage).awaitSingle()
         }
     }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/LibraryManga.kt

@@ -1,8 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.models
-
-class LibraryManga : MangaImpl() {
-
-    var unread: Int = 0
-
-    var category: Int = 0
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt

@@ -1,5 +1,6 @@
 package suwayomi.tachidesk.manga.impl.backup.models
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import org.jetbrains.exposed.sql.ResultRow
 import suwayomi.tachidesk.manga.model.table.MangaTable
 
@@ -25,6 +26,8 @@ open class MangaImpl : Manga {
 
     override var thumbnail_url: String? = null
 
+    override var update_strategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE
+
     override var favorite: Boolean = false
 
     override var last_update: Long = 0

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.backup.proto
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import okio.buffer
 import okio.gzip
 import okio.sink
@@ -59,17 +60,18 @@ object ProtoBackupExport : ProtoBackupBase() {
     private fun backupManga(databaseManga: Query, flags: BackupFlags): List<BackupManga> {
         return databaseManga.map { mangaRow ->
             val backupManga = BackupManga(
-                mangaRow[MangaTable.sourceReference],
+                source = mangaRow[MangaTable.sourceReference],
-                mangaRow[MangaTable.url],
+                url = mangaRow[MangaTable.url],
-                mangaRow[MangaTable.title],
+                title = mangaRow[MangaTable.title],
-                mangaRow[MangaTable.artist],
+                artist = mangaRow[MangaTable.artist],
-                mangaRow[MangaTable.author],
+                author = mangaRow[MangaTable.author],
-                mangaRow[MangaTable.description],
+                description = mangaRow[MangaTable.description],
-                mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
+                genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
-                MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
+                status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
-                mangaRow[MangaTable.thumbnail_url],
+                thumbnailUrl = mangaRow[MangaTable.thumbnail_url],
-                TimeUnit.SECONDS.toMillis(mangaRow[MangaTable.inLibraryAt]),
+                dateAdded = TimeUnit.SECONDS.toMillis(mangaRow[MangaTable.inLibraryAt]),
-                0 // not supported in Tachidesk
+                viewer = 0, // not supported in Tachidesk
+                updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy])
             )
 
             val mangaId = mangaRow[MangaTable.id].value

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -145,6 +145,7 @@ object ProtoBackupImport : ProtoBackupBase() {
                     it[genre] = manga.genre
                     it[status] = manga.status
                     it[thumbnail_url] = manga.thumbnail_url
+                    it[updateStrategy] = manga.update_strategy.name
 
                     it[sourceReference] = manga.source
 
@@ -193,6 +194,7 @@ object ProtoBackupImport : ProtoBackupBase() {
                     it[genre] = manga.genre ?: dbManga[genre]
                     it[status] = manga.status
                     it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
+                    it[updateStrategy] = manga.update_strategy.name
 
                     it[initialized] = dbManga[initialized] || manga.description != null
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt

@@ -1,5 +1,6 @@
 package suwayomi.tachidesk.manga.impl.backup.proto.models
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.protobuf.ProtoNumber
 import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
@@ -35,7 +36,8 @@ data class BackupManga(
     @ProtoNumber(101) var chapterFlags: Int = 0,
     @ProtoNumber(102) var brokenHistory: List<BrokenBackupHistory> = emptyList(),
     @ProtoNumber(103) var viewer_flags: Int? = null,
-    @ProtoNumber(104) var history: List<BackupHistory> = emptyList()
+    @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
+    @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE
 ) {
     fun getMangaImpl(): MangaImpl {
         return MangaImpl().apply {
@@ -52,6 +54,7 @@ data class BackupManga(
             date_added = this@BackupManga.dateAdded
             viewer_flags = this@BackupManga.viewer_flags ?: this@BackupManga.viewer
             chapter_flags = this@BackupManga.chapterFlags
+            update_strategy = this@BackupManga.updateStrategy
         }
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt

@@ -16,7 +16,8 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Page.getPageName
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
+import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
@@ -25,6 +26,7 @@ import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
+import java.io.File
 
 suspend fun getChapterDownloadReady(chapterIndex: Int, mangaId: Int): ChapterDataClass {
     val chapter = ChapterForDownload(chapterIndex, mangaId)
@@ -127,13 +129,16 @@ private class ChapterForDownload(
     }
 
     private fun isNotCompletelyDownloaded(): Boolean {
-        return !(chapterEntry[ChapterTable.isDownloaded] && firstPageExists())
+        return !(
+            chapterEntry[ChapterTable.isDownloaded] &&
+                (firstPageExists() || File(getChapterCbzPath(mangaId, chapterEntry[ChapterTable.id].value)).exists())
+            )
     }
 
     private fun firstPageExists(): Boolean {
         val chapterId = chapterEntry[ChapterTable.id].value
 
-        val chapterDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
 
         println(chapterDir)
         println(getPageName(0))

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/ArchiveProvider.kt

@@ -0,0 +1,89 @@
+package suwayomi.tachidesk.manga.impl.download
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
+import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
+import java.io.File
+import java.io.InputStream
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+import java.util.zip.ZipInputStream
+import java.util.zip.ZipOutputStream
+
+class ArchiveProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) {
+    override fun getImage(index: Int): Pair<InputStream, String> {
+        val cbzPath = getChapterCbzPath(mangaId, chapterId)
+        val zipFile = ZipFile(cbzPath)
+        val zipEntry = zipFile.entries().toList().sortedWith(compareBy({ it.name }, { it.name }))[index]
+        val inputStream = zipFile.getInputStream(zipEntry)
+        val fileType = zipEntry.name.substringAfterLast(".")
+        return Pair(inputStream.buffered(), "image/$fileType")
+    }
+
+    override suspend fun download(
+        download: DownloadChapter,
+        scope: CoroutineScope,
+        step: suspend (DownloadChapter?, Boolean) -> Unit
+    ): Boolean {
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
+        val outputFile = File(getChapterCbzPath(mangaId, chapterId))
+        val chapterFolder = File(chapterDir)
+        if (outputFile.exists()) handleExistingCbzFile(outputFile, chapterFolder)
+
+        FolderProvider(mangaId, chapterId).download(download, scope, step)
+
+        withContext(Dispatchers.IO) {
+            outputFile.createNewFile()
+        }
+
+        ZipOutputStream(outputFile.outputStream()).use { zipOut ->
+            if (chapterFolder.isDirectory) {
+                chapterFolder.listFiles()?.sortedBy { it.name }?.forEach {
+                    val entry = ZipEntry(it.name)
+                    try {
+                        zipOut.putNextEntry(entry)
+                        it.inputStream().use { inputStream ->
+                            inputStream.copyTo(zipOut)
+                        }
+                    } finally {
+                        zipOut.closeEntry()
+                    }
+                }
+            }
+        }
+
+        if (chapterFolder.exists() && chapterFolder.isDirectory) {
+            chapterFolder.deleteRecursively()
+        }
+
+        return true
+    }
+
+    override fun delete(): Boolean {
+        val cbzFile = File(getChapterCbzPath(mangaId, chapterId))
+        if (cbzFile.exists()) return cbzFile.delete()
+        return false
+    }
+
+    private fun handleExistingCbzFile(cbzFile: File, chapterFolder: File) {
+        if (!chapterFolder.exists()) chapterFolder.mkdirs()
+        ZipInputStream(cbzFile.inputStream()).use { zipInputStream ->
+            var zipEntry = zipInputStream.nextEntry
+            while (zipEntry != null) {
+                val file = File(chapterFolder, zipEntry.name)
+                if (!file.exists()) {
+                    file.parentFile.mkdirs()
+                    file.createNewFile()
+                }
+                file.outputStream().use { outputStream ->
+                    zipInputStream.copyTo(outputStream)
+                }
+                zipEntry = zipInputStream.nextEntry
+            }
+        }
+        cbzFile.delete()
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt

@@ -0,0 +1,20 @@
+package suwayomi.tachidesk.manga.impl.download
+
+import kotlinx.coroutines.CoroutineScope
+import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
+import java.io.InputStream
+
+/*
+* Base class for downloaded chapter files provider, example: Folder, Archive
+* */
+abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) {
+    abstract fun getImage(index: Int): Pair<InputStream, String>
+
+    abstract suspend fun download(
+        download: DownloadChapter,
+        scope: CoroutineScope,
+        step: suspend (DownloadChapter?, Boolean) -> Unit
+    ): Boolean
+
+    abstract fun delete(): Boolean
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt

@@ -13,17 +13,13 @@ import kotlinx.coroutines.Job
 import kotlinx.coroutines.cancelAndJoin
 import kotlinx.coroutines.currentCoroutineContext
 import kotlinx.coroutines.ensureActive
-import kotlinx.coroutines.flow.distinctUntilChanged
-import kotlinx.coroutines.flow.launchIn
-import kotlinx.coroutines.flow.onEach
-import kotlinx.coroutines.flow.sample
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import mu.KotlinLogging
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
-import suwayomi.tachidesk.manga.impl.Page.getPageImage
+import suwayomi.tachidesk.manga.impl.ChapterDownloadHelper
 import suwayomi.tachidesk.manga.impl.chapter.getChapterDownloadReady
 import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
 import suwayomi.tachidesk.manga.impl.download.model.DownloadState.Downloading
@@ -95,33 +91,7 @@ class Downloader(
                 download.chapter = getChapterDownloadReady(download.chapterIndex, download.mangaId)
                 step(download, false)
 
-                val pageCount = download.chapter.pageCount
+                ChapterDownloadHelper.download(download.mangaId, download.chapter.id, download, scope, this::step)
-                for (pageNum in 0 until pageCount) {
-                    var pageProgressJob: Job? = null
-                    try {
-                        getPageImage(
-                            mangaId = download.mangaId,
-                            chapterIndex = download.chapterIndex,
-                            index = pageNum,
-                            progressFlow = { flow ->
-                                pageProgressJob = flow
-                                    .sample(100)
-                                    .distinctUntilChanged()
-                                    .onEach {
-                                        download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
-                                        step(null, false) // don't throw on canceled download here since we can't do anything
-                                    }
-                                    .launchIn(scope)
-                            }
-                        ).first.close()
-                    } finally {
-                        // always cancel the page progress job even if it throws an exception to avoid memory leaks
-                        pageProgressJob?.cancel()
-                    }
-                    // TODO: retry on error with 2,4,8 seconds of wait
-                    download.progress = ((pageNum + 1).toFloat()) / pageCount
-                    step(download, false)
-                }
                 download.state = Finished
                 transaction {
                     ChapterTable.update({ (ChapterTable.manga eq download.mangaId) and (ChapterTable.sourceOrder eq download.chapterIndex) }) {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt

@@ -0,0 +1,87 @@
+package suwayomi.tachidesk.manga.impl.download
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.FlowPreview
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.flow.distinctUntilChanged
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.flow.sample
+import suwayomi.tachidesk.manga.impl.Page
+import suwayomi.tachidesk.manga.impl.Page.getPageName
+import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+
+/*
+* Provides downloaded files when pages were downloaded into folders
+* */
+class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) {
+    override fun getImage(index: Int): Pair<InputStream, String> {
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
+        val folder = File(chapterDir)
+        folder.mkdirs()
+        val file = folder.listFiles()?.sortedBy { it.name }?.get(index)
+        val fileType = file!!.name.substringAfterLast(".")
+        return Pair(FileInputStream(file).buffered(), "image/$fileType")
+    }
+
+    @OptIn(FlowPreview::class)
+    override suspend fun download(
+        download: DownloadChapter,
+        scope: CoroutineScope,
+        step: suspend (DownloadChapter?, Boolean) -> Unit
+    ): Boolean {
+        val pageCount = download.chapter.pageCount
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
+        val folder = File(chapterDir)
+        folder.mkdirs()
+
+        for (pageNum in 0 until pageCount) {
+            var pageProgressJob: Job? = null
+            val fileName = getPageName(pageNum) // might have to change this to index stored in database
+            if (isExistingFile(folder, fileName)) continue
+            try {
+                Page.getPageImage(
+                    mangaId = download.mangaId,
+                    chapterIndex = download.chapterIndex,
+                    index = pageNum
+                ) { flow ->
+                    pageProgressJob = flow
+                        .sample(100)
+                        .distinctUntilChanged()
+                        .onEach {
+                            download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
+                            step(null, false) // don't throw on canceled download here since we can't do anything
+                        }
+                        .launchIn(scope)
+                }.first.use { image ->
+                    val filePath = "$chapterDir/$fileName"
+                    ImageResponse.saveImage(filePath, image)
+                }
+            } finally {
+                // always cancel the page progress job even if it throws an exception to avoid memory leaks
+                pageProgressJob?.cancel()
+            }
+            // TODO: retry on error with 2,4,8 seconds of wait
+            download.progress = ((pageNum + 1).toFloat()) / pageCount
+            step(download, false)
+        }
+        return true
+    }
+
+    override fun delete(): Boolean {
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
+        return File(chapterDir).deleteRecursively()
+    }
+
+    private fun isExistingFile(folder: File, fileName: String): Boolean {
+        val existingFile = folder.listFiles { file ->
+            file.isFile && file.name.startsWith(fileName)
+        }?.firstOrNull()
+        return existingFile?.exists() == true
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt

@@ -40,7 +40,7 @@ import suwayomi.tachidesk.manga.impl.util.PackageTools.getPackageInfo
 import suwayomi.tachidesk.manga.impl.util.PackageTools.loadExtensionSources
 import suwayomi.tachidesk.manga.impl.util.network.await
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import suwayomi.tachidesk.manga.model.table.SourceTable
 import suwayomi.tachidesk.server.ApplicationDirs
@@ -266,16 +266,16 @@ object Extension {
         return installExtension(pkgName)
     }
 
-    suspend fun getExtensionIcon(apkName: String, useCache: Boolean): Pair<InputStream, String> {
+    suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
         val iconUrl = if (apkName == "localSource") {
             ""
         } else {
             transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
         }
 
-        val saveDir = "${applicationDirs.extensionsRoot}/icon"
+        val cacheSaveDir = "${applicationDirs.extensionsRoot}/icon"
 
-        return getImageResponse(saveDir, apkName, useCache) {
+        return getCachedImageResponse(cacheSaveDir, apkName) {
             network.client.newCall(
                 GET(iconUrl)
             ).await()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt

@@ -22,6 +22,7 @@ import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension
 import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import java.util.concurrent.ConcurrentHashMap
+import kotlin.time.Duration.Companion.seconds
 
 object ExtensionsList {
     private val logger = KotlinLogging.logger {}
@@ -29,12 +30,9 @@ object ExtensionsList {
     var lastUpdateCheck: Long = 0
     var updateMap = ConcurrentHashMap<String, OnlineExtension>()
 
-    /** 60,000 milliseconds = 60 seconds */
-    private const val ExtensionUpdateDelayTime = 60 * 1000
-
     suspend fun getExtensionList(): List<ExtensionDataClass> {
-        // update if {ExtensionUpdateDelayTime} seconds has passed or requested offline and database is empty
+        // update if 60 seconds has passed or requested offline and database is empty
-        if (lastUpdateCheck + ExtensionUpdateDelayTime < System.currentTimeMillis()) {
+        if (lastUpdateCheck + 60.seconds.inWholeMilliseconds < System.currentTimeMillis()) {
             logger.debug("Getting extensions list from the internet")
             lastUpdateCheck = System.currentTimeMillis()
 
@@ -80,14 +78,14 @@ object ExtensionsList {
                                 updateMap.putIfAbsent(foundExtension.pkgName, foundExtension)
                             }
                             foundExtension.versionCode < extensionRecord[ExtensionTable.versionCode] -> {
-                                // some how the user installed an invalid version
+                                // somehow the user installed an invalid version
                                 ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) {
                                     it[isObsolete] = true
                                 }
                             }
                         }
                     } else {
-                        // extension is not installed so we can overwrite the data without a care
+                        // extension is not installed, so we can overwrite the data without a care
                         ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) {
                             it[name] = foundExtension.name
                             it[versionName] = foundExtension.versionName
@@ -117,14 +115,14 @@ object ExtensionsList {
             ExtensionTable.selectAll().forEach { extensionRecord ->
                 val foundExtension = foundExtensions.find { it.pkgName == extensionRecord[ExtensionTable.pkgName] }
                 if (foundExtension == null) {
-                    // not in the repo, so this extensions is obsolete
+                    // not in the repo, so these extensions are obsolete
                     if (extensionRecord[ExtensionTable.isInstalled]) {
                         // is installed so we should mark it as obsolete
                         ExtensionTable.update({ ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] }) {
                             it[isObsolete] = true
                         }
                     } else {
-                        // is not installed so we can remove the record without a care
+                        // is not installed, so we can remove the record without a care
                         ExtensionTable.deleteWhere { ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] }
                     }
                 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt

@@ -7,11 +7,12 @@ package suwayomi.tachidesk.manga.impl.extension.github
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.await
 import eu.kanade.tachiyomi.network.parseAs
 import kotlinx.serialization.Serializable
-import okhttp3.Request
+import mu.KotlinLogging
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN
 import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
@@ -19,6 +20,8 @@ import uy.kohesive.injekt.injectLazy
 
 object ExtensionGithubApi {
     private const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
+    private const val FALLBACK_REPO_URL_PREFIX = "https://gcore.jsdelivr.net/gh/tachiyomiorg/tachiyomi-extensions@repo/"
+    private val logger = KotlinLogging.logger {}
 
     @Serializable
     private data class ExtensionJsonObject(
@@ -42,13 +45,26 @@ object ExtensionGithubApi {
         val baseUrl: String
     )
 
-    suspend fun findExtensions(): List<OnlineExtension> {
+    private var requiresFallbackSource = false
-        val request = Request.Builder()
-            .url("$REPO_URL_PREFIX/index.min.json")
-            .build()
 
-        return client.newCall(request)
+    suspend fun findExtensions(): List<OnlineExtension> {
-            .await()
+        val githubResponse = if (requiresFallbackSource) {
+            null
+        } else {
+            try {
+                client.newCall(GET("${REPO_URL_PREFIX}index.min.json")).await()
+            } catch (e: Throwable) {
+                logger.error(e) { "Failed to get extensions from GitHub" }
+                requiresFallbackSource = true
+                null
+            }
+        }
+
+        val response = githubResponse ?: run {
+            client.newCall(GET("${FALLBACK_REPO_URL_PREFIX}index.min.json")).await()
+        }
+
+        return response
             .parseAs<List<ExtensionJsonObject>>()
             .toExtensions()
     }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt

@@ -4,7 +4,7 @@ import kotlinx.coroutines.flow.StateFlow
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 
 interface IUpdater {
-    fun addMangaToQueue(manga: MangaDataClass)
+    fun addMangasToQueue(mangas: List<MangaDataClass>)
     val status: StateFlow<UpdateStatus>
     fun reset()
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -14,9 +14,12 @@ import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.update
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import mu.KotlinLogging
 import suwayomi.tachidesk.manga.impl.Chapter
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.server.serverConfig
 import java.util.concurrent.ConcurrentHashMap
 
 class Updater : IUpdater {
@@ -27,18 +30,29 @@ class Updater : IUpdater {
     override val status = _status.asStateFlow()
 
     private val tracker = ConcurrentHashMap<Int, UpdateJob>()
-    private var updateChannel = createUpdateChannel()
+    private val updateChannels = ConcurrentHashMap<String, Channel<UpdateJob>>()
+
+    private val semaphore = Semaphore(serverConfig.maxParallelUpdateRequests)
+
+    private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
+        return updateChannels.getOrPut(source) {
+            logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" }
+            createUpdateChannel()
+        }
+    }
 
     private fun createUpdateChannel(): Channel<UpdateJob> {
         val channel = Channel<UpdateJob>(Channel.UNLIMITED)
         channel.consumeAsFlow()
             .onEach { job ->
-                _status.value = UpdateStatus(
+                semaphore.withPermit {
-                    process(job),
+                    _status.value = UpdateStatus(
-                    tracker.any { (_, job) ->
+                        process(job),
-                        job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
+                        tracker.any { (_, job) ->
-                    }
+                            job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
-                )
+                        }
+                    )
+                }
             }
             .catch { logger.error(it) { "Error during updates" } }
             .launchIn(scope)
@@ -49,7 +63,7 @@ class Updater : IUpdater {
         tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
         _status.update { UpdateStatus(tracker.values.toList(), true) }
         tracker[job.manga.id] = try {
-            logger.info { "Updating ${job.manga.title}" }
+            logger.info { "Updating \"${job.manga.title}\" (source: ${job.manga.sourceId})" }
             Chapter.getChapterList(job.manga.id, true)
             job.copy(status = JobStatus.COMPLETE)
         } catch (e: Exception) {
@@ -60,19 +74,24 @@ class Updater : IUpdater {
         return tracker.values.toList()
     }
 
-    override fun addMangaToQueue(manga: MangaDataClass) {
+    override fun addMangasToQueue(mangas: List<MangaDataClass>) {
+        mangas.forEach { tracker[it.id] = UpdateJob(it) }
+        _status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) }
+        mangas.forEach { addMangaToQueue(it) }
+    }
+
+    private fun addMangaToQueue(manga: MangaDataClass) {
+        val updateChannel = getOrCreateUpdateChannelFor(manga.sourceId)
         scope.launch {
             updateChannel.send(UpdateJob(manga))
         }
-        tracker[manga.id] = UpdateJob(manga)
-        _status.update { UpdateStatus(tracker.values.toList(), true) }
     }
 
     override fun reset() {
         scope.coroutineContext.cancelChildren()
         tracker.clear()
         _status.update { UpdateStatus() }
-        updateChannel.cancel()
+        updateChannels.forEach { (_, channel) -> channel.cancel() }
-        updateChannel = createUpdateChannel()
+        updateChannels.clear()
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.kodein.di.DI
@@ -21,17 +22,16 @@ import java.io.File
 
 private val applicationDirs by DI.global.instance<ApplicationDirs>()
 
-fun getMangaDir(mangaId: Int): String {
+private fun getMangaDir(mangaId: Int): String {
-    val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
+    val mangaEntry = getMangaEntry(mangaId)
     val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
 
-    val sourceDir = source.toString()
+    val sourceDir = SafePath.buildValidFilename(source.toString())
     val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
-
+    return "$sourceDir/$mangaDir"
-    return "${applicationDirs.mangaDownloadsRoot}/$sourceDir/$mangaDir"
 }
 
-fun getChapterDir(mangaId: Int, chapterId: Int): String {
+private fun getChapterDir(mangaId: Int, chapterId: Int): String {
     val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.first() }
 
     val chapterDir = SafePath.buildValidFilename(
@@ -44,9 +44,21 @@ fun getChapterDir(mangaId: Int, chapterId: Int): String {
     return getMangaDir(mangaId) + "/$chapterDir"
 }
 
+fun getChapterDownloadPath(mangaId: Int, chapterId: Int): String {
+    return applicationDirs.mangaDownloadsRoot + "/" + getChapterDir(mangaId, chapterId)
+}
+
+fun getChapterCbzPath(mangaId: Int, chapterId: Int): String {
+    return getChapterDownloadPath(mangaId, chapterId) + ".cbz"
+}
+
+fun getChapterCachePath(mangaId: Int, chapterId: Int): String {
+    return applicationDirs.tempMangaCacheRoot + "/" + getChapterDir(mangaId, chapterId)
+}
+
 /** return value says if rename/move was successful */
 fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean {
-    val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
+    val mangaEntry = getMangaEntry(mangaId)
     val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
 
     val sourceDir = source.toString()
@@ -66,3 +78,7 @@ fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean {
         true
     }
 }
+
+private fun getMangaEntry(mangaId: Int): ResultRow {
+    return transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt

@@ -40,8 +40,8 @@ object PackageTools {
     const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class"
     const val METADATA_SOURCE_FACTORY = "tachiyomi.extension.factory"
     const val METADATA_NSFW = "tachiyomi.extension.nsfw"
-    const val LIB_VERSION_MIN = 1.2
+    const val LIB_VERSION_MIN = 1.3
-    const val LIB_VERSION_MAX = 1.3
+    const val LIB_VERSION_MAX = 1.4
 
     private const val officialSignature = "7ce04da7773d41b489f4693a366c36bcd0a11fc39b547168553c285bd7348e23" // inorichi's key
     private const val unofficialSignature = "64feb21075ba97ebc9cc981243645b331595c111cef1b0d084236a0403b00581" // ArMor's key

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt

@@ -18,6 +18,7 @@ object ImageResponse {
         return FileInputStream(path).buffered()
     }
 
+    /** find file with name when file extension is not known */
     fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
         val target = "$fileName."
         File(directoryPath).listFiles().orEmpty().forEach { file ->
@@ -28,8 +29,17 @@ object ImageResponse {
         return null
     }
 
-    /** fetch a cached image response, calls `fetcher` if cache fails */
+    /**
+     * Get a cached image response
+     *
+     * Note: The caller should also call [clearCachedImage] when appropriate
+     *
+     * @param cacheSavePath where to save the cached image. Caller should decide to use perma cache or temp cache (OS temp dir)
+     * @param fileName what the saved cache file should be named
+     */
     suspend fun getCachedImageResponse(saveDir: String, fileName: String, fetcher: suspend () -> Response): Pair<InputStream, String> {
+        File(saveDir).mkdirs()
+
         val cachedFile = findFileNameStartingWith(saveDir, fileName)
         val filePath = "$saveDir/$fileName"
         if (cachedFile != null) {
@@ -43,19 +53,7 @@ object ImageResponse {
         val response = fetcher()
 
         if (response.code == 200) {
-            val tmpSavePath = "$filePath.tmp"
+            val (actualSavePath, imageType) = saveImage(filePath, response.body!!.byteStream())
-            val tmpSaveFile = File(tmpSavePath)
-            response.body!!.source().saveTo(tmpSaveFile)
-
-            // find image type
-            val imageType = response.headers["content-type"]
-                ?: ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime
-                ?: "image/jpeg"
-
-            val actualSavePath = "$filePath.${imageType.substringAfter("/")}"
-
-            tmpSaveFile.renameTo(File(actualSavePath))
-
             return pathToInputStream(actualSavePath) to imageType
         } else {
             response.closeQuietly()
@@ -63,36 +61,26 @@ object ImageResponse {
         }
     }
 
+    /** Save image safely */
+    fun saveImage(filePath: String, image: InputStream): Pair<String, String> {
+        val tmpSavePath = "$filePath.tmp"
+        val tmpSaveFile = File(tmpSavePath)
+        image.use { input -> tmpSaveFile.outputStream().use { output -> input.copyTo(output) } }
+
+        // find image type
+        val imageType = ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime
+            ?: "image/jpeg"
+
+        val actualSavePath = "$filePath.${imageType.substringAfter("/")}"
+
+        tmpSaveFile.renameTo(File(actualSavePath))
+        return Pair(actualSavePath, imageType)
+    }
+
     fun clearCachedImage(saveDir: String, fileName: String) {
         val cachedFile = findFileNameStartingWith(saveDir, fileName)
         cachedFile?.also {
             File(it).delete()
         }
     }
-
-    suspend fun getNoCacheImageResponse(fetcher: suspend () -> Response): Pair<InputStream, String> {
-        val response = fetcher()
-
-        if (response.code == 200) {
-            val responseBytes = response.body!!.bytes()
-
-            // find image type
-            val imageType = response.headers["content-type"]
-                ?: ImageUtil.findImageType { responseBytes.inputStream() }?.mime
-                ?: "image/jpeg"
-
-            return responseBytes.inputStream() to imageType
-        } else {
-            response.closeQuietly()
-            throw Exception("request error! ${response.code}")
-        }
-    }
-
-    suspend fun getImageResponse(saveDir: String, fileName: String, useCache: Boolean = false, fetcher: suspend () -> Response): Pair<InputStream, String> {
-        return if (useCache) {
-            getCachedImageResponse(saveDir, fileName, fetcher)
-        } else {
-            getNoCacheImageResponse(fetcher)
-        }
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt

@@ -1,5 +1,7 @@
 package suwayomi.tachidesk.manga.model.dataclass
 
+import com.fasterxml.jackson.annotation.JsonValue
+
 /*
  * Copyright (C) Contributors to the Suwayomi project
  *
@@ -7,10 +9,20 @@ package suwayomi.tachidesk.manga.model.dataclass
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+enum class IncludeInUpdate(@JsonValue val value: Int) {
+    EXCLUDE(0), INCLUDE(1), UNSET(-1);
+
+    companion object {
+        fun fromValue(value: Int) = IncludeInUpdate.values().find { it.value == value } ?: UNSET
+    }
+}
+
 data class CategoryDataClass(
     val id: Int,
     val order: Int,
     val name: String,
     val default: Boolean,
+    val size: Int,
+    val includeInUpdate: IncludeInUpdate,
     val meta: Map<String, String> = emptyMap()
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt

@@ -35,6 +35,9 @@ data class ChapterDataClass(
     /** the date we fist saw this chapter*/
     val fetchedAt: Long,
 
+    /** the website url of this chapter*/
+    val realUrl: String? = null,
+
     /** is chapter downloaded */
     val downloaded: Boolean,
 

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.model.dataclass
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import suwayomi.tachidesk.manga.impl.util.lang.trimAll
 import suwayomi.tachidesk.manga.model.table.MangaStatus
 import java.time.Instant
@@ -18,6 +19,7 @@ data class MangaDataClass(
     val url: String,
     val title: String,
     val thumbnailUrl: String? = null,
+    val thumbnailUrlLastFetched: Long = 0,
 
     val initialized: Boolean = false,
 
@@ -37,10 +39,13 @@ data class MangaDataClass(
     var lastFetchedAt: Long? = 0,
     var chaptersLastFetchedAt: Long? = 0,
 
+    var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE,
+
     val freshData: Boolean = false,
     var unreadCount: Long? = null,
     var downloadCount: Long? = null,
     var chapterCount: Long? = null,
+    var lastReadAt: Long? = null,
     var lastChapterRead: ChapterDataClass? = null,
 
     val age: Long? = if (lastFetchedAt == null) 0 else Instant.now().epochSecond.minus(lastFetchedAt),

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryMetaTable.kt

@@ -12,7 +12,7 @@ import org.jetbrains.exposed.sql.ReferenceOption
 import suwayomi.tachidesk.manga.model.table.CategoryMetaTable.ref
 
 /**
- * Metadata storage for clients, about Chapter with id == [ref].
+ * Metadata storage for clients, about Category with id == [ref].
  */
 object CategoryMetaTable : IntIdTable() {
     val key = varchar("key", 256)

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt

@@ -11,11 +11,13 @@ import org.jetbrains.exposed.dao.id.IntIdTable
 import org.jetbrains.exposed.sql.ResultRow
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
+import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 
 object CategoryTable : IntIdTable() {
     val name = varchar("name", 64)
     val order = integer("order").default(0)
     val isDefault = bool("is_default").default(false)
+    val includeInUpdate = integer("include_in_update").default(IncludeInUpdate.UNSET.value)
 }
 
 fun CategoryTable.toDataClass(categoryEntry: ResultRow) = CategoryDataClass(
@@ -23,5 +25,7 @@ fun CategoryTable.toDataClass(categoryEntry: ResultRow) = CategoryDataClass(
     categoryEntry[order],
     categoryEntry[name],
     categoryEntry[isDefault],
+    Category.getCategorySize(categoryEntry[id].value),
+    IncludeInUpdate.fromValue(categoryEntry[includeInUpdate]),
     Category.getCategoryMetaMap(categoryEntry[id].value)
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt

@@ -13,6 +13,7 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.manga.impl.Chapter.getChapterMetaMap
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
+import suwayomi.tachidesk.manga.model.table.MangaTable.nullable
 
 object ChapterTable : IntIdTable() {
     val url = varchar("url", 2048)
@@ -29,6 +30,9 @@ object ChapterTable : IntIdTable() {
 
     val sourceOrder = integer("source_order")
 
+    /** the real url of a chapter used for the "open in WebView" feature */
+    val realUrl = varchar("real_url", 2048).nullable()
+
     val isDownloaded = bool("is_downloaded").default(false)
 
     val pageCount = integer("page_count").default(-1)
@@ -38,21 +42,22 @@ object ChapterTable : IntIdTable() {
 
 fun ChapterTable.toDataClass(chapterEntry: ResultRow) =
     ChapterDataClass(
-        chapterEntry[id].value,
+        id = chapterEntry[id].value,
-        chapterEntry[url],
+        url = chapterEntry[url],
-        chapterEntry[name],
+        name = chapterEntry[name],
-        chapterEntry[date_upload],
+        uploadDate = chapterEntry[date_upload],
-        chapterEntry[chapter_number],
+        chapterNumber = chapterEntry[chapter_number],
-        chapterEntry[scanlator],
+        scanlator = chapterEntry[scanlator],
-        chapterEntry[manga].value,
+        mangaId = chapterEntry[manga].value,
-        chapterEntry[isRead],
+        read = chapterEntry[isRead],
-        chapterEntry[isBookmarked],
+        bookmarked = chapterEntry[isBookmarked],
-        chapterEntry[lastPageRead],
+        lastPageRead = chapterEntry[lastPageRead],
-        chapterEntry[lastReadAt],
+        lastReadAt = chapterEntry[lastReadAt],
-        chapterEntry[sourceOrder],
+        index = chapterEntry[sourceOrder],
-        chapterEntry[fetchedAt],
+        fetchedAt = chapterEntry[fetchedAt],
-        chapterEntry[isDownloaded],
+        realUrl = chapterEntry[realUrl],
-        chapterEntry[pageCount],
+        downloaded = chapterEntry[isDownloaded],
-        transaction { ChapterTable.select { manga eq chapterEntry[manga].value }.count().toInt() },
+        pageCount = chapterEntry[pageCount],
-        getChapterMetaMap(chapterEntry[id])
+        chapterCount = transaction { ChapterTable.select { manga eq chapterEntry[manga].value }.count().toInt() },
+        meta = getChapterMetaMap(chapterEntry[id])
     )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.model.table
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import org.jetbrains.exposed.dao.id.IntIdTable
 import org.jetbrains.exposed.sql.ResultRow
 import suwayomi.tachidesk.manga.impl.Manga.getMangaMetaMap
@@ -28,6 +29,7 @@ object MangaTable : IntIdTable() {
 
     val status = integer("status").default(SManga.UNKNOWN)
     val thumbnail_url = varchar("thumbnail_url", 2048).nullable()
+    val thumbnailUrlLastFetched = long("thumbnail_url_last_fetched").default(0)
 
     val inLibrary = bool("in_library").default(false)
     val defaultCategory = bool("default_category").default(true)
@@ -41,30 +43,34 @@ object MangaTable : IntIdTable() {
 
     val lastFetchedAt = long("last_fetched_at").default(0)
     val chaptersLastFetchedAt = long("chapters_last_fetched_at").default(0)
+
+    val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name)
 }
 
 fun MangaTable.toDataClass(mangaEntry: ResultRow) =
     MangaDataClass(
-        mangaEntry[this.id].value,
+        id = mangaEntry[this.id].value,
-        mangaEntry[sourceReference].toString(),
+        sourceId = mangaEntry[sourceReference].toString(),
 
-        mangaEntry[url],
+        url = mangaEntry[url],
-        mangaEntry[title],
+        title = mangaEntry[title],
-        proxyThumbnailUrl(mangaEntry[this.id].value),
+        thumbnailUrl = proxyThumbnailUrl(mangaEntry[this.id].value),
+        thumbnailUrlLastFetched = mangaEntry[thumbnailUrlLastFetched],
 
-        mangaEntry[initialized],
+        initialized = mangaEntry[initialized],
 
-        mangaEntry[artist],
+        artist = mangaEntry[artist],
-        mangaEntry[author],
+        author = mangaEntry[author],
-        mangaEntry[description],
+        description = mangaEntry[description],
-        mangaEntry[genre].toGenreList(),
+        genre = mangaEntry[genre].toGenreList(),
-        Companion.valueOf(mangaEntry[status]).name,
+        status = Companion.valueOf(mangaEntry[status]).name,
-        mangaEntry[inLibrary],
+        inLibrary = mangaEntry[inLibrary],
-        mangaEntry[inLibraryAt],
+        inLibraryAt = mangaEntry[inLibraryAt],
         meta = getMangaMetaMap(mangaEntry[id].value),
         realUrl = mangaEntry[realUrl],
         lastFetchedAt = mangaEntry[lastFetchedAt],
-        chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt]
+        chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt],
+        updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy])
     )
 
 enum class MangaStatus(val value: Int) {

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -19,15 +19,9 @@ class ServerConfig(config: Config, moduleName: String = MODULE_NAME) : SystemPro
 
     // proxy
     val socksProxyEnabled: Boolean by overridableConfig
-
     val socksProxyHost: String by overridableConfig
     val socksProxyPort: String by overridableConfig
 
-    // misc
-    val debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config)
-    val systemTrayEnabled: Boolean by overridableConfig
-    val downloadsPath: String by overridableConfig
-
     // webUI
     val webUIEnabled: Boolean by overridableConfig
     val webUIFlavor: String by overridableConfig
@@ -35,11 +29,26 @@ class ServerConfig(config: Config, moduleName: String = MODULE_NAME) : SystemPro
     val webUIInterface: String by overridableConfig
     val electronPath: String by overridableConfig
 
+    // downloader
+    val downloadAsCbz: Boolean by overridableConfig
+    val downloadsPath: String by overridableConfig
+
+    // updater
+    val maxParallelUpdateRequests: Int by overridableConfig
+
     // Authentication
     val basicAuthEnabled: Boolean by overridableConfig
     val basicAuthUsername: String by overridableConfig
     val basicAuthPassword: String by overridableConfig
 
+    // misc
+    val debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config)
+    val systemTrayEnabled: Boolean by overridableConfig
+
+    // Webview
+    val webviewEnabled: Boolean by overridableConfig
+    val undetectedChromePath: String by overridableConfig
+
     companion object {
         fun register(config: Config) = ServerConfig(config.getConfig(MODULE_NAME))
     }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt

@@ -36,13 +36,17 @@ import java.util.Locale
 private val logger = KotlinLogging.logger {}
 
 class ApplicationDirs(
-    val dataRoot: String = ApplicationRootDir
+    val dataRoot: String = ApplicationRootDir,
+    val tempRoot: String = "${System.getProperty("java.io.tmpdir")}/Tachidesk"
 ) {
+    val cacheRoot = System.getProperty("java.io.tmpdir") + "/tachidesk"
     val extensionsRoot = "$dataRoot/extensions"
     val thumbnailsRoot = "$dataRoot/thumbnails"
     val mangaDownloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" }
     val localMangaRoot = "$dataRoot/local"
     val webUIRoot = "$dataRoot/webUI"
+
+    val tempMangaCacheRoot = "$tempRoot/manga-cache"
 }
 
 val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() }

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0022_MangaThumbnailLastFetched.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0022_MangaThumbnailLastFetched : AddColumnMigration(
+    "Manga",
+    "thumbnail_url_last_fetched",
+    "BIGINT",
+    "0"
+)

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0023_CategoryMetaRefFix.kt

@@ -0,0 +1,35 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.SQLMigration
+import org.jetbrains.exposed.sql.transactions.TransactionManager
+
+@Suppress("ClassName", "unused")
+class M0023_CategoryMetaRefFix : SQLMigration() {
+    fun String.toSqlName(): String =
+        TransactionManager.defaultDatabase!!.identifierManager.let {
+            it.quoteIfNecessary(
+                it.inProperCase(this)
+            )
+        }
+
+    private val CategoryMetaTable by lazy { "CategoryMeta".toSqlName() }
+    private val CategoryRefColumn by lazy { "category_ref".toSqlName() }
+    private val CategoryTable by lazy { "Category".toSqlName() }
+
+    override val sql by lazy {
+        // Incorrectly referenced in M0021
+        """
+        ALTER TABLE $CategoryMetaTable DROP COLUMN $CategoryRefColumn;
+        ALTER TABLE $CategoryMetaTable ADD COLUMN $CategoryRefColumn INT DEFAULT 0;   
+        ALTER TABLE $CategoryMetaTable ADD FOREIGN KEY ($CategoryRefColumn) 
+                                    REFERENCES $CategoryTable(ID) ON DELETE CASCADE;
+        """
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0024_MangaUpdateStrategy.kt

@@ -0,0 +1,19 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
+
+@Suppress("ClassName", "unused")
+class M0024_MangaUpdateStrategy : AddColumnMigration(
+    "Manga",
+    "update_strategy",
+    "VARCHAR(256)",
+    "'${UpdateStrategy.ALWAYS_UPDATE.name}'"
+)

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0025_ChapterRealUrl.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0025_ChapterRealUrl : AddColumnMigration(
+    "Chapter",
+    "real_url",
+    "VARCHAR(2048)",
+    "NULL"
+)

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0026_CategoryIncludeInUpdate.kt

@@ -0,0 +1,19 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
+
+@Suppress("ClassName", "unused")
+class M0026_CategoryIncludeInUpdate : AddColumnMigration(
+    "Category",
+    "include_in_update",
+    "INT",
+    IncludeInUpdate.UNSET.value.toString()
+)

server/src/main/kotlin/suwayomi/tachidesk/server/util/Browser.kt

@@ -7,7 +7,7 @@ package suwayomi.tachidesk.server.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import dorkbox.util.Desktop
+import dorkbox.desktop.Desktop
 import suwayomi.tachidesk.server.serverConfig
 
 object Browser {

server/src/main/kotlin/suwayomi/tachidesk/server/util/Chromium.kt

@@ -0,0 +1,58 @@
+package suwayomi.tachidesk.server.util
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.decodeFromStream
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import net.harawata.appdirs.AppDirsFactory
+import java.nio.file.FileSystems
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+import kotlin.io.path.Path
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.createDirectories
+import kotlin.io.path.exists
+import kotlin.io.path.notExists
+import kotlin.streams.asSequence
+
+object Chromium {
+    @OptIn(ExperimentalSerializationApi::class)
+    @JvmStatic
+    fun preinstall(platformDir: String) {
+        val loader = Thread.currentThread().contextClassLoader
+        val resource = loader.getResource("driver/$platformDir/package/browsers.json") ?: return
+        val json = resource.openStream().use {
+            Json.decodeFromStream<JsonObject>(it)
+        }
+        val revision = json["browsers"]?.jsonArray
+            ?.find { it.jsonObject["name"]?.jsonPrimitive?.contentOrNull == "chromium" }
+            ?.jsonObject
+            ?.get("revision")
+            ?.jsonPrimitive
+            ?.contentOrNull
+            ?: return
+
+        val playwrightDir = AppDirsFactory.getInstance().getUserDataDir("ms-playwright", null, null)
+        val chromiumZip = Path(".").resolve("bin/chromium.zip")
+        val chromePath = Path(playwrightDir).resolve("chromium-$revision")
+        if (chromePath.exists() || chromiumZip.notExists()) return
+        chromePath.createDirectories()
+
+        FileSystems.newFileSystem(chromiumZip, null as ClassLoader?).use {
+            val src = it.getPath("/")
+            Files.walk(src)
+                .asSequence()
+                .forEach { source ->
+                    Files.copy(
+                        source,
+                        chromePath.resolve(source.absolutePathString().removePrefix("/")),
+                        StandardCopyOption.REPLACE_EXISTING
+                    )
+                }
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/server/util/SystemTray.kt

@@ -51,8 +51,6 @@ object SystemTray {
                 }
             )
 
-            systemTray.installShutdownHook()
-
             return systemTray
         } catch (e: Exception) {
             e.printStackTrace()

server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt

@@ -7,7 +7,6 @@ package suwayomi.tachidesk.server.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import kotlinx.serialization.json.Json
 import mu.KotlinLogging
 import net.lingala.zip4j.ZipFile
 import org.kodein.di.DI
@@ -16,7 +15,6 @@ import org.kodein.di.instance
 import suwayomi.tachidesk.server.ApplicationDirs
 import suwayomi.tachidesk.server.BuildConfig
 import suwayomi.tachidesk.server.serverConfig
-import uy.kohesive.injekt.injectLazy
 import java.io.File
 import java.io.InputStream
 import java.net.HttpURLConnection
@@ -26,7 +24,6 @@ import java.security.MessageDigest
 
 private val logger = KotlinLogging.logger {}
 private val applicationDirs by DI.global.instance<ApplicationDirs>()
-private val json: Json by injectLazy()
 private val tmpDir = System.getProperty("java.io.tmpdir")
 
 private fun ByteArray.toHex(): String = joinToString(separator = "") { eachByte -> "%02x".format(eachByte) }

server/src/main/resources/server-reference.conf

@@ -14,6 +14,13 @@ server.initialOpenInBrowserEnabled = true
 server.webUIInterface = "browser" # "browser" or "electron"
 server.electronPath = ""
 
+# downloader
+server.downloadAsCbz = false
+server.downloadsPath = ""
+
+# updater
+server.maxParallelUpdateRequests = 10 # sets how many sources can be updated in parallel. updates are grouped by source and all mangas of a source are updated synchronously
+
 # Authentication
 server.basicAuthEnabled = false
 server.basicAuthUsername = ""
@@ -22,4 +29,7 @@ server.basicAuthPassword = ""
 # misc
 server.debugLogsEnabled = false
 server.systemTrayEnabled = true
-server.downloadsPath = ""
+
+# Webview
+server.webviewEnabled = false
+server.undetectedChromePath = ""

server/src/test/kotlin/masstest/CloudFlareTest.kt

@@ -0,0 +1,58 @@
+package masstest
+
+import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.test.runTest
+import mu.KotlinLogging
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.TestInstance
+import suwayomi.tachidesk.manga.impl.Source
+import suwayomi.tachidesk.manga.impl.extension.Extension
+import suwayomi.tachidesk.manga.impl.extension.ExtensionsList
+import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
+import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
+import suwayomi.tachidesk.server.applicationSetup
+import suwayomi.tachidesk.test.BASE_PATH
+import suwayomi.tachidesk.test.setLoggingEnabled
+import xyz.nulldev.ts.config.CONFIG_PREFIX
+import java.io.File
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class CloudFlareTest {
+    lateinit var nhentai: HttpSource
+
+    @BeforeAll
+    fun setup() {
+        val dataRoot = File(BASE_PATH).absolutePath
+        System.setProperty("$CONFIG_PREFIX.server.rootDir", dataRoot)
+        applicationSetup()
+        setLoggingEnabled(false)
+
+        runBlocking {
+            val extensions = ExtensionsList.getExtensionList()
+            with(extensions.first { it.name == "NHentai" }) {
+                if (!installed) {
+                    Extension.installExtension(pkgName)
+                } else if (hasUpdate) {
+                    Extension.updateExtension(pkgName)
+                }
+                Unit
+            }
+
+            nhentai = Source.getSourceList()
+                .firstNotNullOf { it.id.toLong().takeIf { it == 3122156392225024195L } }
+                .let(GetCatalogueSource::getCatalogueSourceOrNull) as HttpSource
+        }
+        setLoggingEnabled(true)
+    }
+
+    private val logger = KotlinLogging.logger {}
+
+    @Test
+    fun `test nhentai browse`() = runTest {
+        assert(nhentai.fetchPopularManga(1).awaitSingle().mangas.isNotEmpty()) {
+            "NHentai results were empty"
+        }
+    }
+}

server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt

@@ -13,8 +13,8 @@ class TestUpdater : IUpdater {
     private val _status = MutableStateFlow(UpdateStatus())
     override val status: StateFlow<UpdateStatus> = _status.asStateFlow()
 
-    override fun addMangaToQueue(manga: MangaDataClass) {
+    override fun addMangasToQueue(mangas: List<MangaDataClass>) {
-        updateQueue.add(UpdateJob(manga))
+        mangas.forEach { updateQueue.add(UpdateJob(it)) }
         isRunning = true
         updateStatus()
     }

server/src/test/resources/server-reference.conf

@@ -7,6 +7,12 @@ server.socksProxyEnabled = false
 server.socksProxyHost = ""
 server.socksProxyPort = ""
 
+# downloader
+server.downloadAsCbz = false
+
+# updater
+server.maxParallelUpdateRequests = 10
+
 # misc
 server.debugLogsEnabled = true
 server.systemTrayEnabled = false

settings.gradle.kts

@@ -4,3 +4,5 @@ include("server")
 
 include("AndroidCompat")
 include("AndroidCompat:Config")
+
+enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")