Downloader Rewrite (#437)

* Downloader rewrite
- Rewrite downloader to use coroutines instead of a thread
- Remove unused Page functions
- Add page progress
- Add ProgressResponseBody
- Add support for canceling a download in the middle of downloading
- Fix clear download queue

* Minor fix

* Minor improvements
- notifyAllClients now launches in another thread and only sends new data every second
- Better handling of download queue checker in step()
- Minor improvements and fixes

* Reorder downloads

* Download in parallel by source

* Remove TODO

server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt

@@ -116,13 +116,13 @@ fun Call.asObservableSuccess(): Observable<Response> {
 @Suppress("UNUSED_PARAMETER")
 fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
     val progressClient = newBuilder()
-//        .cache(null)
-//        .addNetworkInterceptor { chain ->
-//            val originalResponse = chain.proceed(chain.request())
-//            originalResponse.newBuilder()
-//                .body(ProgressResponseBody(originalResponse.body!!, listener))
-//                .build()
-//        }
+        .cache(null)
+        .addNetworkInterceptor { chain ->
+            val originalResponse = chain.proceed(chain.request())
+            originalResponse.newBuilder()
+                .body(ProgressResponseBody(originalResponse.body!!, listener))
+                .build()
+        }
         .build()
 
     return progressClient.newCall(request)

server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt

@@ -0,0 +1,44 @@
+package eu.kanade.tachiyomi.network
+
+import okhttp3.MediaType
+import okhttp3.ResponseBody
+import okio.Buffer
+import okio.BufferedSource
+import okio.ForwardingSource
+import okio.Source
+import okio.buffer
+import java.io.IOException
+
+class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
+
+    private val bufferedSource: BufferedSource by lazy {
+        source(responseBody.source()).buffer()
+    }
+
+    override fun contentType(): MediaType? {
+        return responseBody.contentType()
+    }
+
+    override fun contentLength(): Long {
+        return responseBody.contentLength()
+    }
+
+    override fun source(): BufferedSource {
+        return bufferedSource
+    }
+
+    private fun source(source: Source): Source {
+        return object : ForwardingSource(source) {
+            var totalBytesRead = 0L
+
+            @Throws(IOException::class)
+            override fun read(sink: Buffer, byteCount: Long): Long {
+                val bytesRead = super.read(sink, byteCount)
+                // read() returns the number of bytes read, or -1 if this source is exhausted.
+                totalBytesRead += if (bytesRead != -1L) bytesRead else 0
+                progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
+                return bytesRead
+            }
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt

@@ -1,6 +1,5 @@
 package eu.kanade.tachiyomi.source.local.loader
 
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.storage.EpubFile
 import java.io.File
 
@@ -24,7 +23,6 @@ class EpubPageLoader(file: File) : PageLoader {
                 val streamFn = { epub.getInputStream(epub.getEntry(path)!!) }
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt

@@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.source.local.loader
 
 import com.github.junrar.Archive
 import com.github.junrar.rarfile.FileHeader
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import java.io.ByteArrayInputStream
@@ -46,7 +45,6 @@ class RarPageLoader(file: File) : PageLoader {
 
                     ReaderPage(i).apply {
                         stream = streamFn
-                        status = Page.READY
                     }
                 }
         }
@@ -58,7 +56,6 @@ class RarPageLoader(file: File) : PageLoader {
 
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt

@@ -1,6 +1,5 @@
 package eu.kanade.tachiyomi.source.local.loader
 
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import java.io.File
@@ -24,7 +23,6 @@ class ZipPageLoader(file: File) : PageLoader {
                 val streamFn = { zip.getInputStream(entry) }
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt

@@ -2,7 +2,8 @@ package eu.kanade.tachiyomi.source.model
 
 import android.net.Uri
 import eu.kanade.tachiyomi.network.ProgressListener
-import rx.subjects.Subject
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asStateFlow
 
 open class Page(
     val index: Int,
@@ -11,48 +12,17 @@ open class Page(
     @Transient var uri: Uri? = null // Deprecated but can't be deleted due to extensions
 ) : ProgressListener {
 
-    val number: Int
-        get() = index + 1
-
-    @Transient
-    @Volatile
-    var status: Int = 0
-        set(value) {
-            field = value
-            statusSubject?.onNext(value)
-            statusCallback?.invoke(this)
-        }
-
-    @Transient
-    @Volatile
-    var progress: Int = 0
-        set(value) {
-            field = value
-            statusCallback?.invoke(this)
-        }
-
-    @Transient
-    private var statusSubject: Subject<Int, Int>? = null
-
-    @Transient
-    private var statusCallback: ((Page) -> Unit)? = null
+    private val _progress = MutableStateFlow(0)
+    val progress = _progress.asStateFlow()
 
     override fun update(bytesRead: Long, contentLength: Long, done: Boolean) {
-        progress = if (contentLength > 0) {
+        _progress.value = if (contentLength > 0) {
             (100 * bytesRead / contentLength).toInt()
         } else {
             -1
         }
     }
 
-    fun setStatusSubject(subject: Subject<Int, Int>?) {
-        this.statusSubject = subject
-    }
-
-    fun setStatusCallback(f: ((Page) -> Unit)?) {
-        statusCallback = f
-    }
-
     companion object {
         const val QUEUE = 0
         const val LOAD_PAGE = 1

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSourceFetcher.kt

@@ -4,9 +4,7 @@ import eu.kanade.tachiyomi.source.model.Page
 import rx.Observable
 
 fun HttpSource.getImageUrl(page: Page): Observable<Page> {
-    page.status = Page.LOAD_PAGE
     return fetchImageUrl(page)
-        .doOnError { page.status = Page.ERROR }
         .onErrorReturn { null }
         .doOnNext { page.imageUrl = it }
         .map { page }