multi-user experiment

* Update debhelper-compat to 13

* Re-enable debian release

* Re-enable debian release

.github/workflows/issue_moderator.yml

@@ -1,24 +1,35 @@
-name: Issue closer
+name: Issue moderator
 
 on:
   issues:
     types: [opened, edited, reopened]
+  issue_comment:
+    types: [created]
 
 jobs:
   autoclose:
     runs-on: ubuntu-latest
     steps:
-      - name: Autoclose issues
+      - name: Moderate issues
-        uses: arkon/issue-closer-action@v3.0
+        uses: tachiyomiorg/issue-moderator-action@v1
         with:
           repo-token: ${{ github.token }}
-          rules: |
+          duplicate-check-enabled: true
+          duplicate-check-label: Source request
+          existing-check-enabled: true
+          existing-check-label: Source request
+          auto-close-rules: |
             [
               {
                 "type": "title",
-                "regex": ".*<short description>*",
+                "regex": ".*<short description>.*",
                 "message": "You did not fill out the description in the title"
               },
+              {
+                "type": "title",
+                "regex": ".*(<|>)+.*",
+                "message": "You did not remove Angle brackets(< and >) from the title"
+              },
               {
                 "type": "body",
                 "regex": ".*DELETE THIS SECTION IF YOU HAVE READ AND ACKNOWLEDGED IT.*",
@@ -26,7 +37,7 @@ jobs:
               },
               {
                 "type": "body",
-                "regex": "(Tachidesk version|Server Operating System|Server Desktop Environment|Server JVM version|Client Operating System|Client Web Browser):.*(\\(Example:|<usually).*",
+                "regex": ".*(Tachidesk version|Server Operating System|Server Desktop Environment|Server JVM version|Client Operating System|Client Web Browser):.*(\\(Example:|<usually).*",
                 "message": "The requested information was not filled out"
               },
               {

AndroidCompat/build.gradle.kts

@@ -25,4 +25,13 @@ dependencies {
 
     // Android version of SimpleDateFormat
     implementation("com.ibm.icu:icu4j:72.1")
+
+    // OpenJDK lacks native JPEG encoder and native WEBP decoder
+    implementation("com.twelvemonkeys.common:common-lang:3.9.4")
+    implementation("com.twelvemonkeys.common:common-io:3.9.4")
+    implementation("com.twelvemonkeys.common:common-image:3.9.4")
+    implementation("com.twelvemonkeys.imageio:imageio-core:3.9.4")
+    implementation("com.twelvemonkeys.imageio:imageio-metadata:3.9.4")
+    implementation("com.twelvemonkeys.imageio:imageio-jpeg:3.4.1")
+    implementation("com.twelvemonkeys.imageio:imageio-webp:3.9.4")
 }

AndroidCompat/src/main/java/android/graphics/Bitmap.java

@@ -0,0 +1,129 @@
+package android.graphics;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
+import javax.imageio.IIOImage;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageWriteParam;
+import javax.imageio.ImageWriter;
+import javax.imageio.stream.ImageOutputStream;
+
+public final class Bitmap {
+    private int width;
+    private int height;
+    private BufferedImage image;
+
+    public Bitmap(BufferedImage image) {
+        this.image = image;
+        this.width = image.getWidth();
+        this.height = image.getHeight();
+    }
+
+    public BufferedImage getImage() {
+        return image;
+    }
+
+    public int getHeight() {
+        return height;
+    }
+
+    public int getWidth() {
+        return width;
+    }
+
+    public enum CompressFormat {
+        JPEG          (0),
+        PNG           (1),
+        WEBP          (2),
+        WEBP_LOSSY    (3),
+        WEBP_LOSSLESS (4);
+
+        CompressFormat(int nativeInt) {
+            this.nativeInt = nativeInt;
+        }
+
+        final int nativeInt;
+    }
+
+    public enum Config {
+        ALPHA_8(1),
+        RGB_565(3),
+        ARGB_4444(4),
+        ARGB_8888(5),
+        RGBA_F16(6),
+        HARDWARE(7),
+        RGBA_1010102(8);
+
+        final int nativeInt;
+
+        private static Config sConfigs[] = {
+            null, ALPHA_8, null, RGB_565, ARGB_4444, ARGB_8888, RGBA_F16, HARDWARE, RGBA_1010102
+        };
+
+        Config(int ni) {
+            this.nativeInt = ni;
+        }
+
+        static Config nativeToConfig(int ni) {
+            return sConfigs[ni];
+        }
+    }
+
+    public static Bitmap createBitmap(int width, int height, Config config) {
+        BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+        return new Bitmap(image);
+    }
+
+    public boolean compress(CompressFormat format, int quality, OutputStream stream) {
+        if (stream == null) {
+            throw new NullPointerException();
+        }
+
+        if (quality < 0 || quality > 100) {
+            throw new IllegalArgumentException("quality must be 0..100");
+        }
+        float qualityFloat = ((float) quality) / 100;
+
+        String formatString = "";
+        if (format == Bitmap.CompressFormat.PNG) {
+            formatString = "png";
+        } else if (format == Bitmap.CompressFormat.JPEG) {
+            formatString = "jpg";
+        } else {
+            throw new IllegalArgumentException("unsupported compression format!");
+        }
+
+        Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName(formatString);
+        if (!writers.hasNext()) {
+            throw new IllegalStateException("no image writers found for this format!");
+        }
+        ImageWriter writer = (ImageWriter) writers.next();
+
+        ImageOutputStream ios;
+        try {
+            ios = ImageIO.createImageOutputStream(stream);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        writer.setOutput(ios);
+
+        ImageWriteParam param = writer.getDefaultWriteParam();
+        if (formatString == "jpg") {
+            param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
+            param.setCompressionQuality(qualityFloat);
+        }
+
+        try {
+             writer.write(null, new IIOImage(image, null, null), param);
+             ios.close();
+             writer.dispose();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return true;
+    }
+}

AndroidCompat/src/main/java/android/graphics/BitmapFactory.java

@@ -0,0 +1,51 @@
+package android.graphics;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import javax.imageio.ImageIO;
+import javax.imageio.ImageReader;
+import javax.imageio.stream.ImageInputStream;
+
+public class BitmapFactory {
+    public static Bitmap decodeStream(InputStream inputStream) {
+        Bitmap bitmap = null;
+
+        try {
+            ImageInputStream imageInputStream = ImageIO.createImageInputStream(inputStream);
+            Iterator<ImageReader> imageReaders = ImageIO.getImageReaders(imageInputStream);
+
+            if (!imageReaders.hasNext()) {
+                throw new IllegalArgumentException("no reader for image");
+            }
+
+            ImageReader imageReader = imageReaders.next();
+            imageReader.setInput(imageInputStream);
+
+            BufferedImage image = imageReader.read(0, imageReader.getDefaultReadParam());
+            bitmap = new Bitmap(image);
+
+            imageReader.dispose();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return bitmap;
+    }
+
+    public static Bitmap decodeByteArray(byte[] data, int offset, int length) {
+        Bitmap bitmap = null;
+
+        ByteArrayInputStream byteArrayStream = new ByteArrayInputStream(data);
+        try {
+            BufferedImage image = ImageIO.read(byteArrayStream);
+            bitmap = new Bitmap(image);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return bitmap;
+    }
+}

AndroidCompat/src/main/java/android/graphics/Canvas.java

@@ -0,0 +1,21 @@
+package android.graphics;
+
+import java.awt.Graphics2D;
+import java.awt.image.BufferedImage;
+import javax.imageio.ImageIO;
+
+public final class Canvas {
+    private BufferedImage canvasImage;
+    private Graphics2D canvas;
+
+    public Canvas(Bitmap bitmap) {
+        canvasImage = bitmap.getImage();
+        canvas = canvasImage.createGraphics();
+    }
+
+    public void drawBitmap(Bitmap sourceBitmap, Rect src, Rect dst, Paint paint) {        
+        BufferedImage sourceImage = sourceBitmap.getImage();
+        BufferedImage sourceImageCropped = sourceImage.getSubimage(src.left, src.top, src.getWidth(), src.getHeight());
+        canvas.drawImage(sourceImageCropped, null, dst.left, dst.top);
+    }
+}

AndroidCompat/src/main/java/android/graphics/Rect.java

@@ -0,0 +1,122 @@
+package android.graphics;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public final class Rect {
+    int left;
+    int top;
+    int right;
+    int bottom;
+
+    private static final class UnflattenHelper {
+        private static final Pattern FLATTENED_PATTERN = Pattern.compile(
+            "(-?\\d+) (-?\\d+) (-?\\d+) (-?\\d+)");
+
+        static Matcher getMatcher(String str) {
+            return FLATTENED_PATTERN.matcher(str);
+        }
+    }
+
+    public Rect() {
+    }
+
+    public Rect(int left, int top, int right, int bottom) {
+        this.left = left;
+        this.top = top;
+        this.right = right;
+        this.bottom = bottom;
+    }
+
+    public Rect(Rect r) {
+        if (r == null) {
+            this.left = 0;
+            this.top = 0;
+            this.right = 0;
+            this.bottom = 0;
+        } else {
+            this.left = left;
+            this.top = top;
+            this.right = right;
+            this.bottom = bottom;
+        }
+    }
+
+    public final int getWidth() {
+        return right - left;
+    }
+
+    public final int getHeight() {
+        return bottom - top;
+    }
+
+    public static Rect unflattenFromString(String str) {
+        if (str.isEmpty()) {
+            return null;
+        }
+
+        Matcher matcher = UnflattenHelper.getMatcher(str);
+        if (!matcher.matches()) {
+            return null;
+        }
+
+        return new Rect(Integer.parseInt(matcher.group(1)),
+                        Integer.parseInt(matcher.group(2)),
+                        Integer.parseInt(matcher.group(3)),
+                        Integer.parseInt(matcher.group(4)));
+    }
+
+    public String toShortString() {
+        return toShortString(new StringBuilder(32));
+    }
+    
+    public String toShortString(StringBuilder sb) {
+        sb.setLength(0);
+        sb.append('['); sb.append(left); sb.append(',');
+        sb.append(top); sb.append("]["); sb.append(right);
+        sb.append(','); sb.append(bottom); sb.append(']');
+        return sb.toString();
+    }
+
+    public String flattenToString() {
+        StringBuilder sb = new StringBuilder(32);
+        sb.append(left);
+        sb.append(' ');
+        sb.append(top);
+        sb.append(' ');
+        sb.append(right);
+        sb.append(' ');
+        sb.append(bottom);
+        return sb.toString();
+    }
+
+    public void writeToParcel(Parcel out, int flags) {
+        out.writeInt(left);
+        out.writeInt(top);
+        out.writeInt(right);
+        out.writeInt(bottom);
+    }
+
+    public static final Parcelable.Creator<Rect> CREATOR = new Parcelable.Creator<Rect>() {
+        @Override
+        public Rect createFromParcel(Parcel in) {
+            Rect r = new Rect();
+            r.readFromParcel(in);
+            return r;
+        }
+
+        @Override
+        public Rect[] newArray(int size) {
+            return new Rect[size];
+        }
+    };
+
+    public void readFromParcel(Parcel in) {
+        left = in.readInt();
+        top = in.readInt();
+        right = in.readInt();
+        bottom = in.readInt();
+    }
+}

CHANGELOG.md

@@ -1,32 +1,37 @@
 # Server: v0.6.6 + WebUI: r963
 ## TL;DR
-- N/A
+- Batch actions for chapters
+- Improved the downloader
+- WebUI changes:
+    - Support for chapter actions
+    - a lot of code cleanup
+    - some bugfixes
 
 ## Tachidesk-Server Changelog
 - (r1114) fix broken links (by @AriaMoradi)
 - (r1115) fix more broken stuff (by @AriaMoradi)
 - (r1116) fix more broken stuff (by @AriaMoradi)
 - (r1117) fix more broken stuff (by @AriaMoradi)
-- (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393)) 83997633+vedantmgoyal2009@users.noreply.github.com
+- (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393) by @vedantmgoyal2009)
-- (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396)) 30526233+voltrare@users.noreply.github.com
+- (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396) by @vedantmgoyal2009)
-- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397)) mahor1221@pm.me
+- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397) by @mahor1221)
 - (r1121) bump version (by @AriaMoradi)
 - (r1122) Update Changelog (by @AriaMoradi)
-- (r1123) Add libc++-dev ([#405](https://github.com/Suwayomi/Tachidesk-Server/pull/405)) mahor1221@pm.me
+- (r1123) Add libc++-dev ([#405](https://github.com/Suwayomi/Tachidesk-Server/pull/405) by @mahor1221)
-- (r1124) Revert back to correct way of handling jre_dir ([#408](https://github.com/Suwayomi/Tachidesk-Server/pull/408)) mahor1221@pm.me
+- (r1124) Revert back to correct way of handling jre_dir ([#408](https://github.com/Suwayomi/Tachidesk-Server/pull/408) by @mahor1221)
-- (r1125) Update winget.yml ([#410](https://github.com/Suwayomi/Tachidesk-Server/pull/410)) 83997633+vedantmgoyal2009@users.noreply.github.com
+- (r1125) Update winget.yml ([#410](https://github.com/Suwayomi/Tachidesk-Server/pull/410) by @vedantmgoyal2009)
-- (r1126) Remove support for Sorayomi web interface ([#414](https://github.com/Suwayomi/Tachidesk-Server/pull/414)) ebbinghaus.marco@gmail.com
+- (r1126) Remove support for Sorayomi web interface ([#414](https://github.com/Suwayomi/Tachidesk-Server/pull/414) by @marcoebbinghaus)
 - (r1127) Fix downloader memory leak ([#418](https://github.com/Suwayomi/Tachidesk-Server/pull/418) by @Syer10)
 - (r1128) Documentation cleanup ([#417](https://github.com/Suwayomi/Tachidesk-Server/pull/417) by @Syer10)
 - (r1129) Updater cleanup and improvements ([#416](https://github.com/Suwayomi/Tachidesk-Server/pull/416) by @Syer10)
-- (r1130) replace quickjs with Mozilla Rhino ([#415](https://github.com/Suwayomi/Tachidesk-Server/pull/415)) 747367352@qq.com
+- (r1130) replace quickjs with Mozilla Rhino ([#415](https://github.com/Suwayomi/Tachidesk-Server/pull/415) by @xhzhe)
 - (r1131) ktlint (by @AriaMoradi)
 - (r1132) move Tachiyomi's BuildConfig to kotlin dir (by @AriaMoradi)
 - (r1133) remove BuildConfig as extensions now use AppInfo (by @AriaMoradi)
 - (r1134) include list of mangas missing source in restore report ([#421](https://github.com/Suwayomi/Tachidesk-Server/pull/421) by @AriaMoradi)
 - (r1135) Update dependencies ([#422](https://github.com/Suwayomi/Tachidesk-Server/pull/422) by @Syer10)
 - (r1136) Lint ([#423](https://github.com/Suwayomi/Tachidesk-Server/pull/423) by @Syer10)
-- (r1137) Fix: Error handling for popular/latest api if pageNum was supplied as zero ([#424](https://github.com/Suwayomi/Tachidesk-Server/pull/424)) anurag4884@gmail.com
+- (r1137) Fix: Error handling for popular/latest api if pageNum was supplied as zero ([#424](https://github.com/Suwayomi/Tachidesk-Server/pull/424) by @meta-boy)
 - (r1138) Add cache control header to manga page response ([#430](https://github.com/Suwayomi/Tachidesk-Server/pull/430) by @martinek)
 - (r1139) add MangaTable.lastFetchedAt and ChapterTable.chaptersLastFetchedAt ([#431](https://github.com/Suwayomi/Tachidesk-Server/pull/431) by @martinek)
 - (r1140) Pre-load meta entries for all chapters for optimization ([#432](https://github.com/Suwayomi/Tachidesk-Server/pull/432) by @martinek)
@@ -82,7 +87,7 @@
 - (r1117) fix more broken stuff (by @AriaMoradi)
 - (r1118) Update winget.yml ([#393](https://github.com/Suwayomi/Tachidesk-Server/pull/393) by @vedantmgoyal2009)
 - (r1119) fix jre path([#396](https://github.com/Suwayomi/Tachidesk-Server/pull/396) by @voltrare)
-- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397)) by @mahor1221)
+- (r1120) Fix deb package ([#397](https://github.com/Suwayomi/Tachidesk-Server/pull/397) by @mahor1221)
 - (r1121) bump version (by @AriaMoradi)
 
 ## Tachidesk-WebUI Changelog

build.gradle.kts

@@ -18,8 +18,8 @@ allprojects {
     repositories {
         mavenCentral()
         google()
-        maven("https://jitpack.io")
         maven("https://github.com/Suwayomi/Tachidesk-Server/raw/android-jar/")
+        maven("https://jitpack.io")
     }
 }
 

scripts/resources/deb/control

@@ -2,7 +2,7 @@ Source: tachidesk-server
 Section: web
 Priority: optional
 Maintainer: Mahor1221 <mahor1221@pm.me>
-Build-Depends: debhelper-compat (= 12), dh-exec
+Build-Depends: debhelper-compat (= 13), dh-exec
 Standards-Version: 4.5.1
 Homepage: https://github.com/Suwayomi/Tachidesk-Server
 

server/build.gradle.kts

@@ -57,7 +57,7 @@ dependencies {
     implementation("com.github.junrar:junrar:7.5.3")
 
     // CloudflareInterceptor
-    implementation("net.sourceforge.htmlunit:htmlunit:2.65.1")
+    implementation("com.microsoft.playwright:playwright:1.28.0")
 
     // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga
     implementation("org.bouncycastle:bcprov-jdk18on:1.72")

server/src/main/java/eu/kanade/tachiyomi/network/interceptor/DriverJar.java

@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package eu.kanade.tachiyomi.network.interceptor;
+
+import com.microsoft.playwright.impl.driver.Driver;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.*;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/*
+  exact copy of https://github.com/microsoft/playwright-java/blob/4d278c391e3c50738ddea6c3e324a4bbbf719d86/driver-bundle/src/main/java/com/microsoft/playwright/impl/driver/jar/DriverJar.java
+  with diff:
+  108a109,116
+  >
+  >   private FileSystem initFileSystem(URI uri) throws IOException {
+  >     try {
+  >       return FileSystems.newFileSystem(uri, Collections.emptyMap());
+  >     } catch (FileSystemAlreadyExistsException e) {
+  >       return null;
+  >     }
+  >   }
+  116c124
+  <     try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? FileSystems.newFileSystem(uri, Collections.emptyMap()) : null) {
+  ---
+  >     try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? initFileSystem(uri) : null) {
+ */
+public class DriverJar extends Driver {
+  private static final String PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD";
+  private static final String SELENIUM_REMOTE_URL = "SELENIUM_REMOTE_URL";
+  static final String PLAYWRIGHT_NODEJS_PATH = "PLAYWRIGHT_NODEJS_PATH";
+  private final Path driverTempDir;
+  private Path preinstalledNodePath;
+
+  public DriverJar() throws IOException {
+    // Allow specifying custom path for the driver installation
+    // See https://github.com/microsoft/playwright-java/issues/728
+    String alternativeTmpdir = System.getProperty("playwright.driver.tmpdir");
+    String prefix = "playwright-java-";
+    driverTempDir = alternativeTmpdir == null
+      ? Files.createTempDirectory(prefix)
+      : Files.createTempDirectory(Paths.get(alternativeTmpdir), prefix);
+    driverTempDir.toFile().deleteOnExit();
+    String nodePath = System.getProperty("playwright.nodejs.path");
+    if (nodePath != null) {
+      preinstalledNodePath = Paths.get(nodePath);
+      if (!Files.exists(preinstalledNodePath)) {
+        throw new RuntimeException("Invalid Node.js path specified: " + nodePath);
+      }
+    }
+    logMessage("created DriverJar: " + driverTempDir);
+  }
+
+  @Override
+  protected void initialize(Boolean installBrowsers) throws Exception {
+    if (preinstalledNodePath == null && env.containsKey(PLAYWRIGHT_NODEJS_PATH)) {
+      preinstalledNodePath = Paths.get(env.get(PLAYWRIGHT_NODEJS_PATH));
+      if (!Files.exists(preinstalledNodePath)) {
+        throw new RuntimeException("Invalid Node.js path specified: " + preinstalledNodePath);
+      }
+    } else if (preinstalledNodePath != null) {
+      // Pass the env variable to the driver process.
+      env.put(PLAYWRIGHT_NODEJS_PATH, preinstalledNodePath.toString());
+    }
+    extractDriverToTempDir();
+    logMessage("extracted driver from jar to " + driverPath());
+    if (installBrowsers)
+      installBrowsers(env);
+  }
+
+  private void installBrowsers(Map<String, String> env) throws IOException, InterruptedException {
+    String skip = env.get(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD);
+    if (skip == null) {
+      skip = System.getenv(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD);
+    }
+    if (skip != null && !"0".equals(skip) && !"false".equals(skip)) {
+      System.out.println("Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set");
+      return;
+    }
+    if (env.get(SELENIUM_REMOTE_URL) != null || System.getenv(SELENIUM_REMOTE_URL) != null) {
+      logMessage("Skipping browsers download because `SELENIUM_REMOTE_URL` env variable is set");
+      return;
+    }
+    Path driver = driverPath();
+    if (!Files.exists(driver)) {
+      throw new RuntimeException("Failed to find driver: " + driver);
+    }
+    ProcessBuilder pb = createProcessBuilder();
+    pb.command().add("install");
+    pb.redirectError(ProcessBuilder.Redirect.INHERIT);
+    pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+    Process p = pb.start();
+    boolean result = p.waitFor(10, TimeUnit.MINUTES);
+    if (!result) {
+      p.destroy();
+      throw new RuntimeException("Timed out waiting for browsers to install");
+    }
+    if (p.exitValue() != 0) {
+      throw new RuntimeException("Failed to install browsers, exit code: " + p.exitValue());
+    }
+  }
+
+  private static boolean isExecutable(Path filePath) {
+    String name = filePath.getFileName().toString();
+    return name.endsWith(".sh") || name.endsWith(".exe") || !name.contains(".");
+  }
+
+
+  private FileSystem initFileSystem(URI uri) throws IOException {
+    try {
+      return FileSystems.newFileSystem(uri, Collections.emptyMap());
+    } catch (FileSystemAlreadyExistsException e) {
+      return null;
+    }
+  }
+  void extractDriverToTempDir() throws URISyntaxException, IOException {
+    ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+    URI originalUri = classloader.getResource(
+      "driver/" + platformDir()).toURI();
+    URI uri = maybeExtractNestedJar(originalUri);
+
+    // Create zip filesystem if loading from jar.
+    try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? initFileSystem(uri) : null) {
+      Path srcRoot = Paths.get(uri);
+      // jar file system's .relativize gives wrong results when used with
+      // spring-boot-maven-plugin, convert to the default filesystem to
+      // have predictable results.
+      // See https://github.com/microsoft/playwright-java/issues/306
+      Path srcRootDefaultFs = Paths.get(srcRoot.toString());
+      Files.walk(srcRoot).forEach(fromPath -> {
+        if (preinstalledNodePath != null) {
+          String fileName = fromPath.getFileName().toString();
+          if ("node.exe".equals(fileName) || "node".equals(fileName)) {
+            return;
+          }
+        }
+        Path relative = srcRootDefaultFs.relativize(Paths.get(fromPath.toString()));
+        Path toPath = driverTempDir.resolve(relative.toString());
+        try {
+          if (Files.isDirectory(fromPath)) {
+            Files.createDirectories(toPath);
+          } else {
+            Files.copy(fromPath, toPath);
+            if (isExecutable(toPath)) {
+              toPath.toFile().setExecutable(true, true);
+            }
+          }
+          toPath.toFile().deleteOnExit();
+        } catch (IOException e) {
+          throw new RuntimeException("Failed to extract driver from " + uri + ", full uri: " + originalUri, e);
+        }
+      });
+    }
+  }
+
+  private URI maybeExtractNestedJar(final URI uri) throws URISyntaxException {
+    if (!"jar".equals(uri.getScheme())) {
+      return uri;
+    }
+    final String JAR_URL_SEPARATOR = "!/";
+    String[] parts = uri.toString().split("!/");
+    if (parts.length != 3) {
+      return uri;
+    }
+    String innerJar = String.join(JAR_URL_SEPARATOR, parts[0], parts[1]);
+    URI jarUri = new URI(innerJar);
+    try (FileSystem fs = FileSystems.newFileSystem(jarUri, Collections.emptyMap())) {
+      Path fromPath = Paths.get(jarUri);
+      Path toPath = driverTempDir.resolve(fromPath.getFileName().toString());
+      Files.copy(fromPath, toPath);
+      toPath.toFile().deleteOnExit();
+      return new URI("jar:" + toPath.toUri() + JAR_URL_SEPARATOR + parts[2]);
+    } catch (IOException e) {
+      throw new RuntimeException("Failed to extract driver's nested .jar from " + jarUri + "; full uri: " + uri, e);
+    }
+  }
+
+  private static String platformDir() {
+    String name = System.getProperty("os.name").toLowerCase();
+    String arch = System.getProperty("os.arch").toLowerCase();
+
+    if (name.contains("windows")) {
+      return "win32_x64";
+    }
+    if (name.contains("linux")) {
+      if (arch.equals("aarch64")) {
+        return "linux-arm64";
+      } else {
+        return "linux";
+      }
+    }
+    if (name.contains("mac os x")) {
+      return "mac";
+    }
+    throw new RuntimeException("Unexpected os.name value: " + name);
+  }
+
+  @Override
+  protected Path driverDir() {
+    return driverTempDir;
+  }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -39,6 +39,7 @@ class NetworkHelper(context: Context) {
                 .cookieJar(cookieManager)
                 .connectTimeout(30, TimeUnit.SECONDS)
                 .readTimeout(30, TimeUnit.SECONDS)
+                .callTimeout(2, TimeUnit.MINUTES)
                 .addInterceptor(UserAgentInterceptor())
 
             if (serverConfig.debugLogsEnabled) {

server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt

@@ -62,7 +62,7 @@ suspend fun Call.await(): Response {
             object : Callback {
                 override fun onResponse(call: Call, response: Response) {
                     if (!response.isSuccessful) {
-                        continuation.resumeWithException(Exception("HTTP error ${response.code}"))
+                        continuation.resumeWithException(HttpException(response.code))
                         return
                     }
 
@@ -94,7 +94,7 @@ fun Call.asObservableSuccess(): Observable<Response> {
         .doOnNext { response ->
             if (!response.isSuccessful) {
                 response.close()
-                throw Exception("HTTP error ${response.code}")
+                throw HttpException(response.code)
             }
         }
 }
@@ -136,3 +136,5 @@ inline fun <reified T> Response.parseAs(): T {
         return json.decodeFromString(responseBody)
     }
 }
+
+class HttpException(val code: Int) : IllegalStateException("HTTP error $code")

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -1,19 +1,25 @@
 package eu.kanade.tachiyomi.network.interceptor
 
-import com.gargoylesoftware.htmlunit.BrowserVersion
+import com.microsoft.playwright.Browser
-import com.gargoylesoftware.htmlunit.WebClient
+import com.microsoft.playwright.BrowserType.LaunchOptions
-import com.gargoylesoftware.htmlunit.html.HtmlPage
+import com.microsoft.playwright.Page
+import com.microsoft.playwright.Playwright
+import com.microsoft.playwright.PlaywrightException
 import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.interceptor.CFClearance.resolveWithWebView
 import mu.KotlinLogging
 import okhttp3.Cookie
 import okhttp3.HttpUrl
 import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
+import suwayomi.tachidesk.server.ServerConfig
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.IOException
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.DurationUnit
 
-// from TachiWeb-Server
 class CloudflareInterceptor : Interceptor {
     private val logger = KotlinLogging.logger {}
 
@@ -25,20 +31,22 @@ class CloudflareInterceptor : Interceptor {
 
         logger.trace { "CloudflareInterceptor is being used." }
 
-        val response = chain.proceed(originalRequest)
+        val originalResponse = chain.proceed(chain.request())
 
         // Check if Cloudflare anti-bot is on
-        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
+        if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) {
-            return response
+            return originalResponse
         }
 
         logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
 
         return try {
-            response.close()
+            originalResponse.close()
             network.cookies.remove(originalRequest.url.toUri())
 
-            chain.proceed(resolveChallenge(response))
+            val request = resolveWithWebView(originalRequest)
+
+            chain.proceed(request)
         } catch (e: Exception) {
             // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
             // we don't crash the entire app
@@ -46,65 +54,174 @@ class CloudflareInterceptor : Interceptor {
         }
     }
 
-    private fun resolveChallenge(response: Response): Request {
+    companion object {
-        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
+        private val ERROR_CODES = listOf(403, 503)
-            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
+        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-            .build()
+        private val COOKIE_NAMES = listOf("cf_clearance")
-        val convertedCookies = WebClient(browserVersion).use { webClient ->
+    }
-            webClient.options.isThrowExceptionOnFailingStatusCode = false
+}
-            webClient.options.isThrowExceptionOnScriptError = false
+
-            webClient.getPage<HtmlPage>(response.request.url.toString())
+/*
-            webClient.waitForBackgroundJavaScript(10000)
+ * This class is ported from https://github.com/vvanglro/cf-clearance
-            // Challenge solved, process cookies
+ * The original code is licensed under Apache 2.0
-            webClient.cookieManager.cookies.filter {
+*/
-                // Only include Cloudflare cookies
+object CFClearance {
-                it.name.startsWith("__cf") || it.name.startsWith("cf_")
+    private val logger = KotlinLogging.logger {}
-            }.map {
+    private val network: NetworkHelper by injectLazy()
-                // Convert cookies -> OkHttp format
+
-                Cookie.Builder()
+    init {
-                    .domain(it.domain.removePrefix("."))
+        // Fix the default DriverJar issue by providing our own implementation
-                    .expiresAt(it.expires?.time ?: Long.MAX_VALUE)
+        // ref: https://github.com/microsoft/playwright-java/issues/1138
-                    .name(it.name)
+        System.setProperty("playwright.driver.impl", "eu.kanade.tachiyomi.network.interceptor.DriverJar")
-                    .path(it.path)
+    }
-                    .value(it.value).apply {
+
-                        if (it.isHttpOnly) httpOnly()
+    fun resolveWithWebView(originalRequest: Request): Request {
-                        if (it.isSecure) secure()
+        val url = originalRequest.url.toString()
-                    }.build()
+
+        logger.debug { "resolveWithWebView($url)" }
+
+        val cookies = Playwright.create().use { playwright ->
+            playwright.chromium().launch(
+                LaunchOptions()
+                    .setHeadless(false)
+                    .apply {
+                        if (serverConfig.socksProxyEnabled) {
+                            setProxy("socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}")
+                        }
+                    }
+            ).use { browser ->
+                val userAgent = originalRequest.header("User-Agent")
+                if (userAgent != null) {
+                    browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
+                        browserContext.newPage().use { getCookies(it, url) }
+                    }
+                } else {
+                    browser.newPage().use { getCookies(it, url) }
+                }
             }
         }
 
         // Copy cookies to cookie store
-        convertedCookies.forEach {
+        cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
             network.cookies.addAll(
-                HttpUrl.Builder()
+                url = HttpUrl.Builder()
                     .scheme("http")
-                    .host(it.domain)
+                    .host(domain)
                     .build(),
-                listOf(it)
+                cookies = cookies
             )
         }
         // Merge new and existing cookies for this request
         // Find the cookies that we need to merge into this request
-        val convertedForThisRequest = convertedCookies.filter {
+        val convertedForThisRequest = cookies.filter {
-            it.matches(response.request.url)
+            it.matches(originalRequest.url)
         }
         // Extract cookies from current request
         val existingCookies = Cookie.parseAll(
-            response.request.url,
+            originalRequest.url,
-            response.request.headers
+            originalRequest.headers
         )
         // Filter out existing values of cookies that we are about to merge in
         val filteredExisting = existingCookies.filter { existing ->
             convertedForThisRequest.none { converted -> converted.name == existing.name }
         }
+        logger.trace { "Existing cookies" }
+        logger.trace { existingCookies.joinToString("; ") }
         val newCookies = filteredExisting + convertedForThisRequest
-        return response.request.newBuilder()
+        logger.trace { "New cookies" }
-            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+        logger.trace { newCookies.joinToString("; ") }
+        return originalRequest.newBuilder()
+            .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" })
             .build()
     }
 
-    companion object {
+    fun getWebViewUserAgent(): String {
-        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+        return try {
-        private val COOKIE_NAMES = listOf("cf_clearance")
+            Playwright.create().use { playwright ->
+                playwright.chromium().launch(
+                    LaunchOptions()
+                        .setHeadless(true)
+                ).use { browser ->
+                    browser.newPage().use { page ->
+                        val userAgent = page.evaluate("() => {return navigator.userAgent}") as String
+                        logger.debug { "WebView User-Agent is $userAgent" }
+                        return userAgent
                     }
                 }
+            }
+        } catch (e: PlaywrightException) {
+            // Playwright might fail on headless environments like docker
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
+        }
+    }
+
+    private fun getCookies(page: Page, url: String): List<Cookie> {
+        applyStealthInitScripts(page)
+        page.navigate(url)
+        val challengeResolved = waitForChallengeResolve(page)
+
+        return if (challengeResolved) {
+            val cookies = page.context().cookies()
+
+            logger.debug {
+                val userAgent = page.evaluate("() => {return navigator.userAgent}")
+                "Playwright User-Agent is $userAgent"
+            }
+
+            // Convert PlayWright cookies to OkHttp cookies
+            cookies.map {
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.httpOnly) httpOnly()
+                        if (it.secure) secure()
+                    }.build()
+            }
+        } else {
+            logger.debug { "Cloudflare challenge failed to resolve" }
+            throw CloudflareBypassException()
+        }
+    }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18
+    private val stealthInitScripts by lazy {
+        arrayOf(
+            ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
+            ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText()
+        )
+    }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76
+    private fun applyStealthInitScripts(page: Page) {
+        for (script in stealthInitScripts) {
+            page.addInitScript(script)
+        }
+    }
+
+    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21
+    private fun waitForChallengeResolve(page: Page): Boolean {
+        // sometimes the user has to solve the captcha challenge manually, potentially wait a long time
+        val timeoutSeconds = 120
+        repeat(timeoutSeconds) {
+            page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
+            val success = try {
+                page.querySelector("#challenge-form") == null
+            } catch (e: Exception) {
+                logger.debug(e) { "query Error" }
+                false
+            }
+            if (success) return true
+        }
+        return false
+    }
+
+    private class CloudflareBypassException : Exception()
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.source.online
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent
 import eu.kanade.tachiyomi.network.newCallWithProgress
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -372,6 +373,6 @@ abstract class HttpSource : CatalogueSource {
     override fun getFilterList() = FilterList()
 
     companion object {
-        const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63"
+        val DEFAULT_USER_AGENT by lazy { getWebViewUserAgent() }
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -87,8 +87,10 @@ object Manga {
                     it[MangaTable.description] = truncate(sManga.description, 4096)
                     it[MangaTable.genre] = sManga.genre
                     it[MangaTable.status] = sManga.status
-                    if (sManga.thumbnail_url != null && sManga.thumbnail_url.orEmpty().isNotEmpty()) {
+                    if (!sManga.thumbnail_url.isNullOrEmpty() && sManga.thumbnail_url != mangaEntry[MangaTable.thumbnail_url]) {
                         it[MangaTable.thumbnail_url] = sManga.thumbnail_url
+                        it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond
+                        clearMangaThumbnail(mangaId)
                     }
 
                     it[MangaTable.realUrl] = runCatching {
@@ -99,8 +101,6 @@ object Manga {
                 }
             }
 
-            clearMangaThumbnail(mangaId)
-
             mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
 
             MangaDataClass(
@@ -110,6 +110,7 @@ object Manga {
                 mangaEntry[MangaTable.url],
                 mangaEntry[MangaTable.title],
                 proxyThumbnailUrl(mangaId),
+                mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
                 true,
 
@@ -171,6 +172,7 @@ object Manga {
         mangaEntry[MangaTable.url],
         mangaEntry[MangaTable.title],
         proxyThumbnailUrl(mangaId),
+        mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
         true,
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt

@@ -76,6 +76,7 @@ object MangaList {
                         manga.url,
                         manga.title,
                         proxyThumbnailUrl(mangaId),
+                        mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
                         manga.initialized,
 
@@ -101,6 +102,7 @@ object MangaList {
                         manga.url,
                         manga.title,
                         proxyThumbnailUrl(mangaId),
+                        mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
                         true,
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt

@@ -22,6 +22,7 @@ import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension
 import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import java.util.concurrent.ConcurrentHashMap
+import kotlin.time.Duration.Companion.seconds
 
 object ExtensionsList {
     private val logger = KotlinLogging.logger {}
@@ -29,12 +30,9 @@ object ExtensionsList {
     var lastUpdateCheck: Long = 0
     var updateMap = ConcurrentHashMap<String, OnlineExtension>()
 
-    /** 60,000 milliseconds = 60 seconds */
-    private const val ExtensionUpdateDelayTime = 60 * 1000
-
     suspend fun getExtensionList(): List<ExtensionDataClass> {
-        // update if {ExtensionUpdateDelayTime} seconds has passed or requested offline and database is empty
+        // update if 60 seconds has passed or requested offline and database is empty
-        if (lastUpdateCheck + ExtensionUpdateDelayTime < System.currentTimeMillis()) {
+        if (lastUpdateCheck + 60.seconds.inWholeMilliseconds < System.currentTimeMillis()) {
             logger.debug("Getting extensions list from the internet")
             lastUpdateCheck = System.currentTimeMillis()
 
@@ -87,7 +85,7 @@ object ExtensionsList {
                             }
                         }
                     } else {
-                        // extension is not installed so we can overwrite the data without a care
+                        // extension is not installed, so we can overwrite the data without a care
                         ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) {
                             it[name] = foundExtension.name
                             it[versionName] = foundExtension.versionName
@@ -117,14 +115,14 @@ object ExtensionsList {
             ExtensionTable.selectAll().forEach { extensionRecord ->
                 val foundExtension = foundExtensions.find { it.pkgName == extensionRecord[ExtensionTable.pkgName] }
                 if (foundExtension == null) {
-                    // not in the repo, so this extensions is obsolete
+                    // not in the repo, so these extensions are obsolete
                     if (extensionRecord[ExtensionTable.isInstalled]) {
                         // is installed so we should mark it as obsolete
                         ExtensionTable.update({ ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] }) {
                             it[isObsolete] = true
                         }
                     } else {
-                        // is not installed so we can remove the record without a care
+                        // is not installed, so we can remove the record without a care
                         ExtensionTable.deleteWhere { ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] }
                     }
                 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt

@@ -7,11 +7,12 @@ package suwayomi.tachidesk.manga.impl.extension.github
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.await
 import eu.kanade.tachiyomi.network.parseAs
 import kotlinx.serialization.Serializable
-import okhttp3.Request
+import mu.KotlinLogging
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN
 import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
@@ -19,6 +20,8 @@ import uy.kohesive.injekt.injectLazy
 
 object ExtensionGithubApi {
     private const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
+    private const val FALLBACK_REPO_URL_PREFIX = "https://gcore.jsdelivr.net/gh/tachiyomiorg/tachiyomi-extensions@repo/"
+    private val logger = KotlinLogging.logger {}
 
     @Serializable
     private data class ExtensionJsonObject(
@@ -42,13 +45,26 @@ object ExtensionGithubApi {
         val baseUrl: String
     )
 
-    suspend fun findExtensions(): List<OnlineExtension> {
+    private var requiresFallbackSource = false
-        val request = Request.Builder()
-            .url("$REPO_URL_PREFIX/index.min.json")
-            .build()
 
-        return client.newCall(request)
+    suspend fun findExtensions(): List<OnlineExtension> {
-            .await()
+        val githubResponse = if (requiresFallbackSource) {
+            null
+        } else {
+            try {
+                client.newCall(GET("${REPO_URL_PREFIX}index.min.json")).await()
+            } catch (e: Throwable) {
+                logger.error(e) { "Failed to get extensions from GitHub" }
+                requiresFallbackSource = true
+                null
+            }
+        }
+
+        val response = githubResponse ?: run {
+            client.newCall(GET("${FALLBACK_REPO_URL_PREFIX}index.min.json")).await()
+        }
+
+        return response
             .parseAs<List<ExtensionJsonObject>>()
             .toExtensions()
     }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt

@@ -18,6 +18,7 @@ data class MangaDataClass(
     val url: String,
     val title: String,
     val thumbnailUrl: String? = null,
+    val thumbnailUrlLastFetched: Long = 0,
 
     val initialized: Boolean = false,
 

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -28,6 +28,7 @@ object MangaTable : IntIdTable() {
 
     val status = integer("status").default(SManga.UNKNOWN)
     val thumbnail_url = varchar("thumbnail_url", 2048).nullable()
+    val thumbnailUrlLastFetched = long("thumbnail_url_last_fetched").default(0)
 
     val inLibrary = bool("in_library").default(false)
     val defaultCategory = bool("default_category").default(true)
@@ -51,6 +52,7 @@ fun MangaTable.toDataClass(mangaEntry: ResultRow) =
         mangaEntry[url],
         mangaEntry[title],
         proxyThumbnailUrl(mangaEntry[this.id].value),
+        mangaEntry[MangaTable.thumbnailUrlLastFetched],
 
         mangaEntry[initialized],
 

server/src/main/kotlin/suwayomi/tachidesk/server/JavalinSetup.kt

@@ -25,10 +25,10 @@ import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.global.GlobalAPI
 import suwayomi.tachidesk.manga.MangaAPI
+import suwayomi.tachidesk.server.database.DBManager.databaseForUser
 import suwayomi.tachidesk.server.util.Browser
 import suwayomi.tachidesk.server.util.setupWebInterface
 import java.io.IOException
-import java.lang.IllegalArgumentException
 import java.util.concurrent.CompletableFuture
 import kotlin.concurrent.thread
 
@@ -66,6 +66,8 @@ object JavalinSetup {
                     ctx.header("WWW-Authenticate", "Basic")
                     ctx.status(401).json("Unauthorized")
                 } else {
+                    val username = if (serverConfig.basicAuthEnabled) serverConfig.basicAuthUsername else "default-user"
+                    ctx.attribute("db", databaseForUser(username))
                     handler.handle(ctx)
                 }
             }

server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt

@@ -19,27 +19,50 @@ import suwayomi.tachidesk.server.ApplicationDirs
 import suwayomi.tachidesk.server.ServerConfig
 
 object DBManager {
+    private val logger = KotlinLogging.logger {}
 
-    val db by lazy {
+    val masterDB by lazy {
         val applicationDirs by DI.global.instance<ApplicationDirs>()
+
         Database.connect(
-            "jdbc:h2:${applicationDirs.dataRoot}/database",
+            "jdbc:h2:${applicationDirs.dataRoot}/server",
             "org.h2.Driver",
             databaseConfig = DatabaseConfig {
                 useNestedTransactions = true
             }
         )
     }
+
+    fun masterDbUp() {
+        val db = masterDB
+        logger.info { "Initialized masterDb: ${masterDB.url}" }
     }
 
-private val logger = KotlinLogging.logger {}
+    private val usersDatabases = mutableMapOf<String, Database>()
+    fun databaseForUser(username: String) {
+        usersDatabases.getOrElse(username) {
+            val applicationDirs by DI.global.instance<ApplicationDirs>()
 
-fun databaseUp(db: Database = DBManager.db) {
             // call db to initialize the lazy object
             logger.info {
-        "Using ${db.vendor} database version ${db.version}"
+                "initializing database for user $username"
             }
 
-    val migrations = loadMigrationsFrom("suwayomi.tachidesk.server.database.migration", ServerConfig::class.java)
+            val database = Database.connect(
-    runMigrations(migrations)
+                "jdbc:h2:${applicationDirs.dataRoot}/server",
+                "org.h2.Driver",
+                databaseConfig = DatabaseConfig {
+                    useNestedTransactions = true
+                }
+            )
+
+            val migrations =
+                loadMigrationsFrom("suwayomi.tachidesk.server.database.migration", ServerConfig::class.java)
+            runMigrations(migrations, database)
+
+            usersDatabases[username] = database
+
+            database
+        }
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0022_MangaThumbnailLastFetched.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0022_MangaThumbnailLastFetched : AddColumnMigration(
+    "Manga",
+    "thumbnail_url_last_fetched",
+    "BIGINT",
+    "0"
+)

server/src/main/kotlin/suwayomi/tachidesk/server/model/table/User.kt

@@ -0,0 +1,10 @@
+package suwayomi.tachidesk.server.model.table
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+class User

server/src/main/resources/cloudflare-js/canvas.fingerprinting.js

@@ -0,0 +1,28 @@
+(function () {
+    const ORIGINAL_CANVAS = HTMLCanvasElement.prototype[name];
+    Object.defineProperty(HTMLCanvasElement.prototype, name, {
+        "value": function () {
+            var shift = {
+                'r': Math.floor(Math.random() * 10) - 5,
+                'g': Math.floor(Math.random() * 10) - 5,
+                'b': Math.floor(Math.random() * 10) - 5,
+                'a': Math.floor(Math.random() * 10) - 5
+            };
+            var width = this.width,
+                height = this.height,
+                context = this.getContext("2d");
+            var imageData = context.getImageData(0, 0, width, height);
+            for (var i = 0; i < height; i++) {
+                for (var j = 0; j < width; j++) {
+                    var n = ((i * (width * 4)) + (j * 4));
+                    imageData.data[n + 0] = imageData.data[n + 0] + shift.r;
+                    imageData.data[n + 1] = imageData.data[n + 1] + shift.g;
+                    imageData.data[n + 2] = imageData.data[n + 2] + shift.b;
+                    imageData.data[n + 3] = imageData.data[n + 3] + shift.a;
+                }
+            }
+            context.putImageData(imageData, 0, 0);
+            return ORIGINAL_CANVAS.apply(this, arguments);
+        }
+    });
+})(this);

server/src/main/resources/cloudflare-js/chrome.global.js

@@ -0,0 +1,52 @@
+Object.defineProperty(window, 'chrome', {
+    value: new Proxy(window.chrome, {
+        has: (target, key) => true,
+        get: (target, key) => {
+            return {
+                app: {
+                    isInstalled: false,
+                },
+                webstore: {
+                    onInstallStageChanged: {},
+                    onDownloadProgress: {},
+                },
+                runtime: {
+                    PlatformOs: {
+                        MAC: 'mac',
+                        WIN: 'win',
+                        ANDROID: 'android',
+                        CROS: 'cros',
+                        LINUX: 'linux',
+                        OPENBSD: 'openbsd',
+                    },
+                    PlatformArch: {
+                        ARM: 'arm',
+                        X86_32: 'x86-32',
+                        X86_64: 'x86-64',
+                    },
+                    PlatformNaclArch: {
+                        ARM: 'arm',
+                        X86_32: 'x86-32',
+                        X86_64: 'x86-64',
+                    },
+                    RequestUpdateCheckStatus: {
+                        THROTTLED: 'throttled',
+                        NO_UPDATE: 'no_update',
+                        UPDATE_AVAILABLE: 'update_available',
+                    },
+                    OnInstalledReason: {
+                        INSTALL: 'install',
+                        UPDATE: 'update',
+                        CHROME_UPDATE: 'chrome_update',
+                        SHARED_MODULE_UPDATE: 'shared_module_update',
+                    },
+                    OnRestartRequiredReason: {
+                        APP_UPDATE: 'app_update',
+                        OS_UPDATE: 'os_update',
+                        PERIODIC: 'periodic',
+                    },
+                },
+            }
+        }
+    })
+});

server/src/main/resources/cloudflare-js/chrome.plugin.js

@@ -0,0 +1,203 @@
+(function () {
+    const plugin0 = Object.create(Plugin.prototype);
+
+    const mimeType0 = Object.create(MimeType.prototype);
+    const mimeType1 = Object.create(MimeType.prototype);
+    Object.defineProperties(mimeType0, {
+        type: {
+            get: () => 'application/pdf',
+        },
+        suffixes: {
+            get: () => 'pdf',
+        },
+    });
+
+    Object.defineProperties(mimeType1, {
+        type: {
+            get: () => 'text/pdf',
+        },
+        suffixes: {
+            get: () => 'pdf',
+        },
+    });
+
+    Object.defineProperties(plugin0, {
+        name: {
+            get: () => 'Chrome PDF Viewer',
+        },
+        description: {
+            get: () => 'Portable Document Format',
+        },
+        0: {
+            get: () => {
+                return mimeType0;
+            },
+        },
+        1: {
+            get: () => {
+                return mimeType1;
+            },
+        },
+        length: {
+            get: () => 2,
+        },
+        filename: {
+            get: () => 'internal-pdf-viewer',
+        },
+    });
+
+    const plugin1 = Object.create(Plugin.prototype);
+    Object.defineProperties(plugin1, {
+        name: {
+            get: () => 'Chromium PDF Viewer',
+        },
+        description: {
+            get: () => 'Portable Document Format',
+        },
+        0: {
+            get: () => {
+                return mimeType0;
+            },
+        },
+        1: {
+            get: () => {
+                return mimeType1;
+            },
+        },
+        length: {
+            get: () => 2,
+        },
+        filename: {
+            get: () => 'internal-pdf-viewer',
+        },
+    });
+
+    const plugin2 = Object.create(Plugin.prototype);
+    Object.defineProperties(plugin2, {
+        name: {
+            get: () => 'Microsoft Edge PDF Viewer',
+        },
+        description: {
+            get: () => 'Portable Document Format',
+        },
+        0: {
+            get: () => {
+                return mimeType0;
+            },
+        },
+        1: {
+            get: () => {
+                return mimeType1;
+            },
+        },
+        length: {
+            get: () => 2,
+        },
+        filename: {
+            get: () => 'internal-pdf-viewer',
+        },
+    });
+
+    const plugin3 = Object.create(Plugin.prototype);
+    Object.defineProperties(plugin3, {
+        name: {
+            get: () => 'PDF Viewer',
+        },
+        description: {
+            get: () => 'Portable Document Format',
+        },
+        0: {
+            get: () => {
+                return mimeType0;
+            },
+        },
+        1: {
+            get: () => {
+                return mimeType1;
+            },
+        },
+        length: {
+            get: () => 2,
+        },
+        filename: {
+            get: () => 'internal-pdf-viewer',
+        },
+    });
+
+    const plugin4 = Object.create(Plugin.prototype);
+    Object.defineProperties(plugin4, {
+        name: {
+            get: () => 'WebKit built-in PDF',
+        },
+        description: {
+            get: () => 'Portable Document Format',
+        },
+        0: {
+            get: () => {
+                return mimeType0;
+            },
+        },
+        1: {
+            get: () => {
+                return mimeType1;
+            },
+        },
+        length: {
+            get: () => 2,
+        },
+        filename: {
+            get: () => 'internal-pdf-viewer',
+        },
+    });
+
+    const pluginArray = Object.create(PluginArray.prototype);
+
+    pluginArray['0'] = plugin0;
+    pluginArray['1'] = plugin1;
+    pluginArray['2'] = plugin2;
+    pluginArray['3'] = plugin3;
+    pluginArray['4'] = plugin4;
+
+    let refreshValue;
+
+    Object.defineProperties(pluginArray, {
+        length: {
+            get: () => 5,
+        },
+        item: {
+            value: (index) => {
+                if (index > 4294967295) {
+                    index = index % 4294967296;
+                }
+                switch (index) {
+                    case 0:
+                        return plugin3;
+                    case 1:
+                        return plugin0;
+                    case 2:
+                        return plugin1;
+                    case 3:
+                        return plugin2;
+                    case 4:
+                        return plugin4;
+                    default:
+                        break;
+                }
+            },
+        },
+        refresh: {
+            get: () => {
+                return refreshValue;
+            },
+            set: (value) => {
+                refreshValue = value;
+            },
+        },
+    });
+
+    Object.defineProperty(Object.getPrototypeOf(navigator), 'plugins', {
+        get: () => {
+            return pluginArray;
+        },
+    });
+})();

server/src/main/resources/cloudflare-js/chrome.runtime.js

@@ -0,0 +1,170 @@
+(function () {
+  window.chrome = {};
+  window.chrome.app = {
+    InstallState: {
+      DISABLED: 'disabled',
+      INSTALLED: 'installed',
+      NOT_INSTALLED: 'not_installed',
+    },
+    RunningState: {
+      CANNOT_RUN: 'cannot_run',
+      READY_TO_RUN: 'ready_to_run',
+      RUNNING: 'running',
+    },
+    getDetails: () => {
+      '[native code]';
+    },
+    getIsInstalled: () => {
+      '[native code]';
+    },
+    installState: () => {
+      '[native code]';
+    },
+    get isInstalled() {
+      return false;
+    },
+    runningState: () => {
+      '[native code]';
+    },
+  };
+
+  window.chrome.runtime = {
+    OnInstalledReason: {
+      CHROME_UPDATE: 'chrome_update',
+      INSTALL: 'install',
+      SHARED_MODULE_UPDATE: 'shared_module_update',
+      UPDATE: 'update',
+    },
+    OnRestartRequiredReason: {
+      APP_UPDATE: 'app_update',
+      OS_UPDATE: 'os_update',
+      PERIODIC: 'periodic',
+    },
+    PlatformArch: {
+      ARM: 'arm',
+      ARM64: 'arm64',
+      MIPS: 'mips',
+      MIPS64: 'mips64',
+      X86_32: 'x86-32',
+      X86_64: 'x86-64',
+    },
+    PlatformNaclArch: {
+      ARM: 'arm',
+      MIPS: 'mips',
+      MIPS64: 'mips64',
+      X86_32: 'x86-32',
+      X86_64: 'x86-64',
+    },
+    PlatformOs: {
+      ANDROID: 'android',
+      CROS: 'cros',
+      FUCHSIA: 'fuchsia',
+      LINUX: 'linux',
+      MAC: 'mac',
+      OPENBSD: 'openbsd',
+      WIN: 'win',
+    },
+    RequestUpdateCheckStatus: {
+      NO_UPDATE: 'no_update',
+      THROTTLED: 'throttled',
+      UPDATE_AVAILABLE: 'update_available',
+    },
+    connect() {
+      '[native code]';
+    },
+    sendMessage() {
+      '[native code]';
+    },
+    id: undefined,
+  };
+
+  let startE = Date.now();
+  window.chrome.csi = function () {
+    '[native code]';
+    return {
+      startE: startE,
+      onloadT: startE + 281,
+      pageT: 3947.235,
+      tran: 15,
+    };
+  };
+
+  window.chrome.loadTimes = function () {
+    '[native code]';
+    return {
+      get requestTime() {
+        return startE / 1000;
+      },
+      get startLoadTime() {
+        return startE / 1000;
+      },
+      get commitLoadTime() {
+        return startE / 1000 + 0.324;
+      },
+      get finishDocumentLoadTime() {
+        return startE / 1000 + 0.498;
+      },
+      get finishLoadTime() {
+        return startE / 1000 + 0.534;
+      },
+      get firstPaintTime() {
+        return startE / 1000 + 0.437;
+      },
+      get firstPaintAfterLoadTime() {
+        return 0;
+      },
+      get navigationType() {
+        return 'Other';
+      },
+      get wasFetchedViaSpdy() {
+        return true;
+      },
+      get wasNpnNegotiated() {
+        return true;
+      },
+      get npnNegotiatedProtocol() {
+        return 'h3';
+      },
+      get wasAlternateProtocolAvailable() {
+        return false;
+      },
+      get connectionInfo() {
+        return 'h3';
+      },
+    };
+  };
+})();
+
+// Bypass OOPIF test
+(function performance_memory() {
+  const jsHeapSizeLimitInt = 4294705152;
+
+  const total_js_heap_size = 35244183;
+  const used_js_heap_size = [
+    17632315, 17632315, 17632315, 17634847, 17636091, 17636751,
+  ];
+
+  let counter = 0;
+
+  let MemoryInfoProto = Object.getPrototypeOf(performance.memory);
+  Object.defineProperties(MemoryInfoProto, {
+    jsHeapSizeLimit: {
+      get: () => {
+        return jsHeapSizeLimitInt;
+      },
+    },
+    totalJSHeapSize: {
+      get: () => {
+        return total_js_heap_size;
+      },
+    },
+    usedJSHeapSize: {
+      get: () => {
+        if (counter > 5) {
+          counter = 0;
+        }
+        return used_js_heap_size[counter++];
+      },
+    },
+  });
+})();

server/src/main/resources/cloudflare-js/emulate.touch.js

@@ -0,0 +1,3 @@
+Object.defineProperty(navigator, 'maxTouchPoints', {
+    get: () => 1
+});

server/src/main/resources/cloudflare-js/navigator.permissions.js

@@ -0,0 +1,33 @@
+// https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js
+if (!window.Notification) {
+    window.Notification = {
+        permission: 'denied'
+    }
+}
+const originalQuery = window.navigator.permissions.query
+window.navigator.permissions.__proto__.query = parameters =>
+    parameters.name === 'notifications'
+        ? Promise.resolve({state: window.Notification.permission})
+        : originalQuery(parameters)
+const oldCall = Function.prototype.call
+
+function call() {
+    return oldCall.apply(this, arguments)
+}
+
+Function.prototype.call = call
+const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
+const oldToString = Function.prototype.toString
+
+function functionToString() {
+    if (this === window.navigator.permissions.query) {
+        return 'function query() { [native code] }'
+    }
+    if (this === functionToString) {
+        return nativeToStringFunctionString
+    }
+    return oldCall.call(oldToString, this)
+}
+
+// eslint-disable-next-line
+Function.prototype.toString = functionToString

server/src/main/resources/cloudflare-js/navigator.webdriver.js

@@ -0,0 +1,5 @@
+Object.defineProperty(Navigator.prototype, 'webdriver', {
+  get() {
+    return false;
+  },
+});

server/src/test/kotlin/masstest/CloudFlareTest.kt

@@ -0,0 +1,58 @@
+package masstest
+
+import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.test.runTest
+import mu.KotlinLogging
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.TestInstance
+import suwayomi.tachidesk.manga.impl.Source
+import suwayomi.tachidesk.manga.impl.extension.Extension
+import suwayomi.tachidesk.manga.impl.extension.ExtensionsList
+import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
+import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
+import suwayomi.tachidesk.server.applicationSetup
+import suwayomi.tachidesk.test.BASE_PATH
+import suwayomi.tachidesk.test.setLoggingEnabled
+import xyz.nulldev.ts.config.CONFIG_PREFIX
+import java.io.File
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class CloudFlareTest {
+    lateinit var nhentai: HttpSource
+
+    @BeforeAll
+    fun setup() {
+        val dataRoot = File(BASE_PATH).absolutePath
+        System.setProperty("$CONFIG_PREFIX.server.rootDir", dataRoot)
+        applicationSetup()
+        setLoggingEnabled(false)
+
+        runBlocking {
+            val extensions = ExtensionsList.getExtensionList()
+            with(extensions.first { it.name == "NHentai" }) {
+                if (!installed) {
+                    Extension.installExtension(pkgName)
+                } else if (hasUpdate) {
+                    Extension.updateExtension(pkgName)
+                }
+                Unit
+            }
+
+            nhentai = Source.getSourceList()
+                .firstNotNullOf { it.id.toLong().takeIf { it == 3122156392225024195L } }
+                .let(GetCatalogueSource::getCatalogueSourceOrNull) as HttpSource
+        }
+        setLoggingEnabled(true)
+    }
+
+    private val logger = KotlinLogging.logger {}
+
+    @Test
+    fun `test nhentai browse`() = runTest {
+        assert(nhentai.fetchPopularManga(1).awaitSingle().mangas.isNotEmpty()) {
+            "NHentai results were empty"
+        }
+    }
+}