mirror of
https://github.com/Suwayomi/Suwayomi-Server.git
synced 2026-06-30 17:34:39 -05:00
Remote Image Processing (#1684)
* Update ServerConfig.kt * Update ConversionUtil.kt * Update Page.kt * Update ServerConfig.kt fixed deletions caused by ide * Update ServerConfig.kt * Update ServerConfig.kt * Cleanup * Post-processing terminology * More comments * Lint * Add known image mimes * Fix weird mime set/get * Implement different downloadConversions and serveConversions * Lint * Improve Post-Processing massivly * Fix thumbnail build * Use Array for headers * Actually fix headers * Actually fix headers 2 * Manually parse DownloadConversion * Cleanup parse * Fix write * Update TypeName * Optimize imports * Remove header type * Fix build --------- Co-authored-by: Syer10 <syer10@users.noreply.github.com>
This commit is contained in:
@@ -1,21 +1,42 @@
|
||||
package suwayomi.tachidesk.graphql.types
|
||||
|
||||
import kotlin.time.Duration
|
||||
|
||||
// These types belong to SettingsType.kt. However, since that file is auto-generated, these types need to be placed in
|
||||
// a "static" file.
|
||||
|
||||
data class DownloadConversion(
|
||||
class DownloadConversion(
|
||||
val target: String,
|
||||
val compressionLevel: Double? = null,
|
||||
val callTimeout: Duration? = null,
|
||||
val connectTimeout: Duration? = null,
|
||||
val headers: Map<String, String>? = null,
|
||||
)
|
||||
|
||||
interface SettingsDownloadConversion {
|
||||
val mimeType: String
|
||||
val target: String
|
||||
val compressionLevel: Double?
|
||||
val callTimeout: Duration?
|
||||
val connectTimeout: Duration?
|
||||
val headers: List<SettingsDownloadConversionHeader>?
|
||||
}
|
||||
|
||||
class SettingsDownloadConversionType(
|
||||
override val mimeType: String,
|
||||
override val target: String,
|
||||
override val compressionLevel: Double?,
|
||||
override val callTimeout: Duration?,
|
||||
override val connectTimeout: Duration?,
|
||||
override val headers: List<SettingsDownloadConversionHeaderType>?
|
||||
) : SettingsDownloadConversion
|
||||
|
||||
interface SettingsDownloadConversionHeader {
|
||||
val name: String
|
||||
val value: String
|
||||
}
|
||||
|
||||
class SettingsDownloadConversionHeaderType(
|
||||
override val name: String,
|
||||
override val value: String,
|
||||
) : SettingsDownloadConversionHeader
|
||||
|
||||
@@ -4,6 +4,8 @@ import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.protobuf.ProtoNumber
|
||||
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversion
|
||||
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionHeader
|
||||
import kotlin.time.Duration
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
@Serializable
|
||||
@@ -11,4 +13,15 @@ class BackupSettingsDownloadConversionType(
|
||||
@ProtoNumber(1) override val mimeType: String,
|
||||
@ProtoNumber(2) override val target: String,
|
||||
@ProtoNumber(3) override val compressionLevel: Double?,
|
||||
) : SettingsDownloadConversion
|
||||
@ProtoNumber(4) override val callTimeout: Duration?,
|
||||
@ProtoNumber(5) override val connectTimeout: Duration?,
|
||||
@ProtoNumber(6) override val headers: List<BackupSettingsDownloadConversionHeaderType>?
|
||||
) : SettingsDownloadConversion
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
@Serializable
|
||||
class BackupSettingsDownloadConversionHeaderType(
|
||||
@ProtoNumber(1) override val name: String,
|
||||
@ProtoNumber(2) override val value: String
|
||||
|
||||
): SettingsDownloadConversionHeader
|
||||
|
||||
@@ -33,11 +33,13 @@ import suwayomi.tachidesk.graphql.types.DownloadConversion
|
||||
import suwayomi.tachidesk.graphql.types.KoreaderSyncChecksumMethod
|
||||
import suwayomi.tachidesk.graphql.types.KoreaderSyncConflictStrategy
|
||||
import suwayomi.tachidesk.graphql.types.KoreaderSyncLegacyStrategy
|
||||
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionHeaderType
|
||||
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionType
|
||||
import suwayomi.tachidesk.graphql.types.WebUIChannel
|
||||
import suwayomi.tachidesk.graphql.types.WebUIFlavor
|
||||
import suwayomi.tachidesk.graphql.types.WebUIInterface
|
||||
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
|
||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSettingsDownloadConversionHeaderType
|
||||
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSettingsDownloadConversionType
|
||||
import suwayomi.tachidesk.manga.impl.extension.repoMatchRegex
|
||||
import suwayomi.tachidesk.server.settings.BooleanSetting
|
||||
@@ -537,8 +539,8 @@ class ServerConfig(
|
||||
excludeFromBackup = true,
|
||||
)
|
||||
|
||||
val downloadConversions: MutableStateFlow<Map<String, DownloadConversion>> by MapSetting<String, DownloadConversion>(
|
||||
protoNumber = 57,
|
||||
fun createDownloadConversionsMap(protoNumber: Int, key: String) = MapSetting<String, DownloadConversion>(
|
||||
protoNumber = protoNumber,
|
||||
defaultValue = emptyMap(),
|
||||
group = SettingGroup.DOWNLOADER,
|
||||
typeInfo =
|
||||
@@ -559,6 +561,14 @@ class ServerConfig(
|
||||
it.key,
|
||||
it.value.target,
|
||||
it.value.compressionLevel,
|
||||
it.value.callTimeout,
|
||||
it.value.connectTimeout,
|
||||
it.value.headers?.map { header ->
|
||||
SettingsDownloadConversionHeaderType(
|
||||
header.key,
|
||||
header.value,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
},
|
||||
@@ -571,6 +581,11 @@ class ServerConfig(
|
||||
DownloadConversion(
|
||||
target = it.target,
|
||||
compressionLevel = it.compressionLevel,
|
||||
callTimeout = it.callTimeout,
|
||||
connectTimeout = it.connectTimeout,
|
||||
headers = it.headers?.associate { header ->
|
||||
header.name to header.value
|
||||
},
|
||||
)
|
||||
}
|
||||
},
|
||||
@@ -583,6 +598,14 @@ class ServerConfig(
|
||||
it.key,
|
||||
it.value.target,
|
||||
it.value.compressionLevel,
|
||||
it.value.callTimeout,
|
||||
it.value.connectTimeout,
|
||||
it.value.headers?.map { header ->
|
||||
BackupSettingsDownloadConversionHeaderType(
|
||||
header.key,
|
||||
header.value,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
},
|
||||
@@ -590,12 +613,16 @@ class ServerConfig(
|
||||
description =
|
||||
"""
|
||||
map input mime type to conversion information, or "default" for others
|
||||
server.downloadConversions."image/webp" = {
|
||||
target = "image/jpeg" # image type to convert to
|
||||
server.$key."image/webp" = {
|
||||
target = "image/jpeg" # image type to convert to, can also be a url to an external server
|
||||
compressionLevel = 0.8 # quality in range [0,1], leave away to use default compression
|
||||
}
|
||||
""".trimIndent(),
|
||||
)
|
||||
val downloadConversions: MutableStateFlow<Map<String, DownloadConversion>> by createDownloadConversionsMap(
|
||||
protoNumber = 57,
|
||||
key = "downloadConversions"
|
||||
)
|
||||
|
||||
val jwtAudience: MutableStateFlow<String> by StringSetting(
|
||||
protoNumber = 58,
|
||||
@@ -669,6 +696,7 @@ class ServerConfig(
|
||||
typeInfo = SettingsRegistry.PartialTypeInfo(imports = listOf("suwayomi.tachidesk.graphql.types.KoreaderSyncChecksumMethod")),
|
||||
)
|
||||
|
||||
@Suppress("DEPRECATION")
|
||||
@Deprecated("Use koreaderSyncStrategyForward and koreaderSyncStrategyBackward instead")
|
||||
val koreaderSyncStrategy: MutableStateFlow<KoreaderSyncLegacyStrategy> by MigratedConfigValue(
|
||||
protoNumber = 64,
|
||||
@@ -707,15 +735,11 @@ class ServerConfig(
|
||||
),
|
||||
readMigrated = {
|
||||
// This is a best-effort reverse mapping. It's not perfect but covers common cases.
|
||||
when {
|
||||
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.PROMPT &&
|
||||
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.PROMPT -> KoreaderSyncLegacyStrategy.PROMPT
|
||||
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE &&
|
||||
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SILENT
|
||||
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL &&
|
||||
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SEND
|
||||
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE &&
|
||||
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE -> KoreaderSyncLegacyStrategy.RECEIVE
|
||||
when (koreaderSyncStrategyForward.value) {
|
||||
KoreaderSyncConflictStrategy.PROMPT if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.PROMPT -> KoreaderSyncLegacyStrategy.PROMPT
|
||||
KoreaderSyncConflictStrategy.KEEP_REMOTE if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SILENT
|
||||
KoreaderSyncConflictStrategy.KEEP_LOCAL if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SEND
|
||||
KoreaderSyncConflictStrategy.KEEP_REMOTE if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE -> KoreaderSyncLegacyStrategy.RECEIVE
|
||||
else -> KoreaderSyncLegacyStrategy.DISABLED
|
||||
}
|
||||
},
|
||||
@@ -885,6 +909,11 @@ class ServerConfig(
|
||||
description = "Controls the MimeType that Suwayomi sends in OPDS entries for CBZ archives. Also affects global CBZ download. Modern follows recent IANA standard (2017), while LEGACY (deprecated mimetype for .cbz) and COMPATIBLE (deprecated mimetype for all comic archives) might be more compatible with older clients.",
|
||||
)
|
||||
|
||||
val serveConversions: MutableStateFlow<Map<String, DownloadConversion>> by createDownloadConversionsMap(
|
||||
protoNumber = 84,
|
||||
key = "serveConversions"
|
||||
)
|
||||
|
||||
|
||||
|
||||
/** ****************************************************************** **/
|
||||
|
||||
@@ -14,6 +14,7 @@ object ConfigTypeRegistration {
|
||||
|
||||
registerCustomType(MutableStateFlowType())
|
||||
registerCustomType(DurationType())
|
||||
registerCustomType(DownloadConversionType())
|
||||
|
||||
registered = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package suwayomi.tachidesk.server.util
|
||||
|
||||
import com.typesafe.config.Config
|
||||
import com.typesafe.config.ConfigFactory
|
||||
import com.typesafe.config.ConfigValue
|
||||
import io.github.config4k.ClassContainer
|
||||
import io.github.config4k.CustomType
|
||||
import io.github.config4k.extract
|
||||
import io.github.config4k.toConfig
|
||||
import suwayomi.tachidesk.graphql.types.DownloadConversion
|
||||
import kotlin.time.Duration
|
||||
|
||||
class DownloadConversionType : CustomType {
|
||||
override fun parse(
|
||||
clazz: ClassContainer,
|
||||
config: Config,
|
||||
name: String,
|
||||
): Any? {
|
||||
val target = config.extract<String>("$name.target")
|
||||
val compressionLevel = config.extract<Double?>("$name.compressionLevel")
|
||||
val callTimeout = config.extract<Duration?>("$name.callTimeout")
|
||||
val connectTimeout = config.extract<Duration?>("$name.connectTimeout")
|
||||
val headers = config.extract<Map<String, String>?>("$name.headers")
|
||||
|
||||
return DownloadConversion(
|
||||
target = target,
|
||||
compressionLevel = compressionLevel,
|
||||
callTimeout = callTimeout,
|
||||
connectTimeout = connectTimeout,
|
||||
headers = headers,
|
||||
)
|
||||
}
|
||||
|
||||
override fun testParse(clazz: ClassContainer): Boolean =
|
||||
clazz.mapperClass.qualifiedName == "suwayomi.tachidesk.graphql.types.DownloadConversion"
|
||||
|
||||
override fun testToConfig(obj: Any): Boolean = obj is DownloadConversion
|
||||
|
||||
override fun toConfig(
|
||||
obj: Any,
|
||||
name: String,
|
||||
): Config {
|
||||
val conversion = obj as DownloadConversion
|
||||
val builder = ConfigFactory.empty()
|
||||
|
||||
var config =
|
||||
builder
|
||||
.withValue("$name.target", conversion.target.asConfigValue())
|
||||
.withValueIfPresent("$name.compressionLevel", conversion.compressionLevel)
|
||||
.withValueIfPresent("$name.callTimeout", conversion.callTimeout?.toString())
|
||||
.withValueIfPresent("$name.connectTimeout", conversion.connectTimeout?.toString())
|
||||
|
||||
if (conversion.headers != null) {
|
||||
config =
|
||||
config
|
||||
.withValue("$name.headers", conversion.headers.asConfigValue())
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
private fun Config.withValueIfPresent(
|
||||
key: String,
|
||||
value: Any?,
|
||||
): Config =
|
||||
if (value != null) {
|
||||
withValue(key, value.asConfigValue())
|
||||
} else {
|
||||
this
|
||||
}
|
||||
|
||||
private fun Any.asConfigValue(): ConfigValue = toConfig("internal").getValue("internal")
|
||||
}
|
||||
@@ -475,7 +475,7 @@ object MangaController {
|
||||
|
||||
ctx.future {
|
||||
future {
|
||||
Page.getPageImage(
|
||||
Page.getPageImageServe(
|
||||
mangaId = mangaId,
|
||||
chapterIndex = chapterIndex,
|
||||
index = index,
|
||||
|
||||
@@ -10,6 +10,7 @@ package suwayomi.tachidesk.manga.impl
|
||||
import eu.kanade.tachiyomi.source.local.LocalSource
|
||||
import eu.kanade.tachiyomi.source.model.Page
|
||||
import eu.kanade.tachiyomi.source.online.HttpSource
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import libcore.net.MimeUtils
|
||||
import org.jetbrains.exposed.sql.SortOrder
|
||||
@@ -17,6 +18,7 @@ import org.jetbrains.exposed.sql.and
|
||||
import org.jetbrains.exposed.sql.selectAll
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import org.jetbrains.exposed.sql.update
|
||||
import suwayomi.tachidesk.graphql.types.DownloadConversion
|
||||
import suwayomi.tachidesk.manga.impl.util.getChapterCachePath
|
||||
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
|
||||
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
|
||||
@@ -24,14 +26,20 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
|
||||
import suwayomi.tachidesk.manga.model.table.ChapterTable
|
||||
import suwayomi.tachidesk.manga.model.table.MangaTable
|
||||
import suwayomi.tachidesk.manga.model.table.PageTable
|
||||
import suwayomi.tachidesk.server.serverConfig
|
||||
import suwayomi.tachidesk.util.ConversionUtil
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.File
|
||||
import java.io.InputStream
|
||||
import javax.imageio.IIOImage
|
||||
import javax.imageio.ImageIO
|
||||
import javax.imageio.ImageWriteParam
|
||||
import javax.imageio.ImageWriter
|
||||
|
||||
object Page {
|
||||
private val logger = KotlinLogging.logger {}
|
||||
|
||||
/**
|
||||
* A page might have a imageUrl ready from the get go, or we might need to
|
||||
* go an extra step and call fetchImageUrl to get it.
|
||||
@@ -51,7 +59,6 @@ object Page {
|
||||
chapterId: Int? = null,
|
||||
chapterIndex: Int? = null,
|
||||
index: Int,
|
||||
format: String? = null,
|
||||
progressFlow: ((StateFlow<Int>) -> Unit)? = null,
|
||||
): Pair<InputStream, String> {
|
||||
val mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() }
|
||||
@@ -73,7 +80,7 @@ object Page {
|
||||
|
||||
try {
|
||||
if (chapterEntry[ChapterTable.isDownloaded]) {
|
||||
return convertImageResponse(ChapterDownloadHelper.getImage(mangaId, chapterId, index), format)
|
||||
return ChapterDownloadHelper.getImage(mangaId, chapterId, index)
|
||||
}
|
||||
} catch (_: Exception) {
|
||||
// ignore and fetch again
|
||||
@@ -102,15 +109,12 @@ object Page {
|
||||
// is of archive format
|
||||
if (LocalSource.pageCache.containsKey(chapterEntry[ChapterTable.url])) {
|
||||
val pageStream = LocalSource.pageCache[chapterEntry[ChapterTable.url]]!![index]
|
||||
return convertImageResponse(pageStream() to (ImageUtil.findImageType { pageStream() }?.mime ?: "image/jpeg"), format)
|
||||
return pageStream() to (ImageUtil.findImageType { pageStream() }?.mime ?: "image/jpeg")
|
||||
}
|
||||
|
||||
// is of directory format
|
||||
val imageFile = File(tachiyomiPage.imageUrl!!)
|
||||
return convertImageResponse(
|
||||
imageFile.inputStream() to (ImageUtil.findImageType { imageFile.inputStream() }?.mime ?: "image/jpeg"),
|
||||
format,
|
||||
)
|
||||
return imageFile.inputStream() to (ImageUtil.findImageType { imageFile.inputStream() }?.mime ?: "image/jpeg")
|
||||
}
|
||||
|
||||
val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
|
||||
@@ -130,38 +134,210 @@ object Page {
|
||||
val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
|
||||
|
||||
// Note: don't care about invalidating cache because OS cache is not permanent
|
||||
return convertImageResponse(
|
||||
getImageResponse(cacheSaveDir, fileName) {
|
||||
source.getImage(tachiyomiPage)
|
||||
},
|
||||
format,
|
||||
)
|
||||
return getImageResponse(cacheSaveDir, fileName) {
|
||||
source.getImage(tachiyomiPage)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun getPageImageServe(
|
||||
mangaId: Int,
|
||||
chapterIndex: Int,
|
||||
index: Int,
|
||||
format: String? = null,
|
||||
): Pair<InputStream, String> {
|
||||
val (inputStream, mime) =
|
||||
getPageImage(
|
||||
mangaId = mangaId,
|
||||
chapterIndex = chapterIndex,
|
||||
index = index,
|
||||
)
|
||||
val conversions = serverConfig.serveConversions.value
|
||||
val defaultConversion = conversions["default"]
|
||||
val formatConversion = format?.let { DownloadConversion(target = it) }
|
||||
val conversion =
|
||||
formatConversion
|
||||
?: conversions[mime]
|
||||
?: defaultConversion
|
||||
?: return inputStream to mime
|
||||
|
||||
val converted =
|
||||
try {
|
||||
convertImageResponse(
|
||||
image = inputStream,
|
||||
mime = mime,
|
||||
conversion = conversion,
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
logger.error(e) { "Error while post-processing image" }
|
||||
null
|
||||
}
|
||||
return converted?.also { inputStream.close() } ?: (inputStream to mime)
|
||||
}
|
||||
|
||||
suspend fun getPageImageDownload(
|
||||
mangaId: Int,
|
||||
chapterId: Int,
|
||||
index: Int,
|
||||
downloadCacheFolder: File,
|
||||
fileName: String,
|
||||
progressFlow: (StateFlow<Int>) -> Unit,
|
||||
) {
|
||||
val (inputStream, mime) =
|
||||
getPageImage(
|
||||
mangaId = mangaId,
|
||||
chapterId = chapterId,
|
||||
index = index,
|
||||
progressFlow = progressFlow,
|
||||
)
|
||||
val conversions = serverConfig.downloadConversions.value
|
||||
if (conversions.isEmpty() || !downloadCacheFolder.exists()) {
|
||||
inputStream.close()
|
||||
return
|
||||
}
|
||||
val defaultConversion = conversions["default"]
|
||||
val conversion =
|
||||
conversions[mime]
|
||||
?: defaultConversion
|
||||
if (conversion == null) {
|
||||
inputStream.close()
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
val converted =
|
||||
try {
|
||||
convertImageResponse(
|
||||
image = inputStream,
|
||||
mime = mime,
|
||||
conversion = conversion,
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw e
|
||||
} finally {
|
||||
inputStream.close()
|
||||
}
|
||||
|
||||
if (converted != null) {
|
||||
val (convertedStream, convertedMime) = converted
|
||||
val convertedExtension =
|
||||
MimeUtils.guessExtensionFromMimeType(convertedMime)
|
||||
?: convertedMime.substringAfter('/')
|
||||
val convertedPage =
|
||||
File(
|
||||
downloadCacheFolder,
|
||||
"$fileName.$convertedExtension",
|
||||
)
|
||||
|
||||
convertedPage.outputStream().use { outputStream ->
|
||||
convertedStream.use { it.copyTo(outputStream) }
|
||||
}
|
||||
|
||||
val extension =
|
||||
MimeUtils.guessExtensionFromMimeType(mime)
|
||||
?: mime.substringAfter('/')
|
||||
if (extension != convertedExtension) {
|
||||
File(
|
||||
downloadCacheFolder,
|
||||
"$fileName.$extension",
|
||||
).delete()
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logger.warn(e) { "Error while post-processing image" }
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun convertImageResponse(
|
||||
image: Pair<InputStream, String>,
|
||||
format: String? = null,
|
||||
): Pair<InputStream, String> {
|
||||
val imageExtension = MimeUtils.guessExtensionFromMimeType(image.second) ?: image.second.removePrefix("image/")
|
||||
image: InputStream,
|
||||
mime: String,
|
||||
conversion: DownloadConversion,
|
||||
): Pair<InputStream, String>? {
|
||||
// Apply HTTP post-process if configured (complementary with format conversion)
|
||||
if (ConversionUtil.isHttpPostProcess(conversion)) {
|
||||
try {
|
||||
val processedStream =
|
||||
ConversionUtil
|
||||
.imageHttpPostProcess(
|
||||
inputStream = image,
|
||||
mimeType = mime,
|
||||
conversion = conversion,
|
||||
)?.buffered()
|
||||
if (processedStream != null) {
|
||||
val mime =
|
||||
ImageUtil.findImageType(processedStream)?.mime
|
||||
?: "image/jpeg"
|
||||
|
||||
val targetExtension =
|
||||
(if (format != imageExtension) format else null)
|
||||
?: return image
|
||||
return processedStream to mime
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// HTTP post-processing failed, continue with original image
|
||||
logger.warn(e) { "Error while post-processing image" }
|
||||
}
|
||||
return null
|
||||
} else {
|
||||
if (mime == conversion.target) {
|
||||
return null
|
||||
}
|
||||
|
||||
val outStream = ByteArrayOutputStream()
|
||||
val writers = ImageIO.getImageWritersBySuffix(targetExtension)
|
||||
val writer = writers.next()
|
||||
ImageIO.createImageOutputStream(outStream).use { o ->
|
||||
writer.setOutput(o)
|
||||
|
||||
val inImage =
|
||||
ConversionUtil.readImage(image.first, image.second)
|
||||
?: throw NoSuchElementException("No conversion to $targetExtension possible")
|
||||
writer.write(inImage)
|
||||
return convertToFormat(image, mime, conversion)
|
||||
}
|
||||
}
|
||||
|
||||
private fun convertToFormat(
|
||||
inputStream: InputStream,
|
||||
sourceMimeType: String,
|
||||
target: DownloadConversion,
|
||||
): Pair<InputStream, String>? {
|
||||
val outStream = ByteArrayOutputStream()
|
||||
val conversionWriter =
|
||||
getConversionWriter(
|
||||
target.target,
|
||||
target.compressionLevel,
|
||||
)
|
||||
if (conversionWriter == null) {
|
||||
logger.warn { "Conversion aborted: No reader for target format ${target.target}" }
|
||||
return inputStream to sourceMimeType
|
||||
}
|
||||
|
||||
val (writer, writerParams) = conversionWriter
|
||||
try {
|
||||
ImageIO.createImageOutputStream(outStream).use { o ->
|
||||
writer.setOutput(o)
|
||||
|
||||
val inImage =
|
||||
ConversionUtil.readImage(inputStream, sourceMimeType)
|
||||
?: throw NoSuchElementException("No conversion to ${target.target} possible")
|
||||
writer.write(null, IIOImage(inImage, null, null), writerParams)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logger.warn(e) { "Conversion aborted" }
|
||||
return null
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
writer.dispose()
|
||||
val inStream = ByteArrayInputStream(outStream.toByteArray())
|
||||
return Pair(inStream.buffered(), MimeUtils.guessMimeTypeFromExtension(targetExtension) ?: "image/$targetExtension")
|
||||
return Pair(inStream.buffered(), target.target)
|
||||
}
|
||||
|
||||
private fun getConversionWriter(
|
||||
targetMime: String,
|
||||
compressionLevel: Double?,
|
||||
): Pair<ImageWriter, ImageWriteParam>? {
|
||||
val writers = ImageIO.getImageWritersByMIMEType(targetMime)
|
||||
val writer =
|
||||
try {
|
||||
writers.next()
|
||||
} catch (_: NoSuchElementException) {
|
||||
return null
|
||||
}
|
||||
|
||||
val writerParams = writer.defaultWriteParam
|
||||
compressionLevel?.let {
|
||||
writerParams.compressionMode = ImageWriteParam.MODE_EXPLICIT
|
||||
writerParams.compressionQuality = it.toFloat()
|
||||
}
|
||||
|
||||
return writer to writerParams
|
||||
}
|
||||
|
||||
/** converts 0 to "001" */
|
||||
|
||||
@@ -151,10 +151,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
|
||||
|
||||
try {
|
||||
Page
|
||||
.getPageImage(
|
||||
.getPageImageDownload(
|
||||
mangaId = download.mangaId,
|
||||
chapterId = download.chapterId,
|
||||
index = pageNum,
|
||||
downloadCacheFolder,
|
||||
fileName,
|
||||
) { flow ->
|
||||
pageProgressJob =
|
||||
flow
|
||||
@@ -167,8 +169,7 @@ abstract class ChaptersFilesProvider<Type : FileType>(
|
||||
false,
|
||||
) // don't throw on canceled download here since we can't do anything
|
||||
}.launchIn(scope)
|
||||
}.first
|
||||
.close()
|
||||
}
|
||||
} finally {
|
||||
// always cancel the page progress job even if it throws an exception to avoid memory leaks
|
||||
pageProgressJob?.cancel()
|
||||
@@ -188,8 +189,6 @@ abstract class ChaptersFilesProvider<Type : FileType>(
|
||||
},
|
||||
)
|
||||
|
||||
maybeConvertPages(downloadCacheFolder)
|
||||
|
||||
handleSuccessfulDownload()
|
||||
|
||||
// Calculate and save Koreader hash for CBZ files
|
||||
@@ -221,104 +220,4 @@ abstract class ChaptersFilesProvider<Type : FileType>(
|
||||
abstract fun getAsArchiveStream(): Pair<InputStream, Long>
|
||||
|
||||
abstract fun getArchiveSize(): Long
|
||||
|
||||
private fun maybeConvertPages(chapterCacheFolder: File) {
|
||||
val conversions = serverConfig.downloadConversions.value
|
||||
|
||||
if (!chapterCacheFolder.isDirectory || conversions.isEmpty()) {
|
||||
return
|
||||
}
|
||||
|
||||
val pages =
|
||||
chapterCacheFolder
|
||||
.listFiles()
|
||||
.orEmpty()
|
||||
.filter { it.name != COMIC_INFO_FILE }
|
||||
|
||||
val pagesByMimeType =
|
||||
pages
|
||||
.groupBy { MimeUtils.guessMimeTypeFromExtension(it.extension) }
|
||||
.mapValues { it.value.map { it.nameWithoutExtension } }
|
||||
|
||||
logger.debug { "maybeConvertPages: pagesByMimeType= $pagesByMimeType; conversions= $conversions" }
|
||||
|
||||
pages.forEach { page ->
|
||||
val imageType = MimeUtils.guessMimeTypeFromExtension(page.extension) ?: return@forEach
|
||||
|
||||
val defaultConversion = conversions["default"]
|
||||
val conversion = conversions[imageType]
|
||||
val targetConversion = conversion ?: defaultConversion ?: return@forEach
|
||||
|
||||
val (targetMime) = targetConversion
|
||||
val requiresConversion = imageType != targetMime && targetMime != "none"
|
||||
if (!requiresConversion) {
|
||||
return@forEach
|
||||
}
|
||||
|
||||
convertPage(page, targetConversion)
|
||||
}
|
||||
}
|
||||
|
||||
private fun convertPage(
|
||||
page: File,
|
||||
conversion: DownloadConversion,
|
||||
) {
|
||||
val (targetMime, compressionLevel) = conversion
|
||||
|
||||
val targetExtension =
|
||||
MimeUtils.guessExtensionFromMimeType(targetMime) ?: targetMime.removePrefix("image/")
|
||||
|
||||
val convertedPage = File(page.parentFile, page.nameWithoutExtension + "." + targetExtension)
|
||||
|
||||
val conversionWriter = getConversionWriter(targetMime, compressionLevel)
|
||||
if (conversionWriter == null) {
|
||||
logger.warn { "Conversion aborted: No reader for target format $targetMime" }
|
||||
return
|
||||
}
|
||||
|
||||
val (writer, writerParams) = conversionWriter
|
||||
|
||||
val success =
|
||||
try {
|
||||
ImageIO.createImageOutputStream(convertedPage).use { outStream ->
|
||||
writer.setOutput(outStream)
|
||||
|
||||
val inImage = ConversionUtil.readImage(page) ?: return@use false
|
||||
writer.write(null, IIOImage(inImage, null, null), writerParams)
|
||||
|
||||
true
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logger.warn(e) { "Conversion aborted: for image $page" }
|
||||
false
|
||||
}
|
||||
writer.dispose()
|
||||
|
||||
if (success) {
|
||||
page.delete()
|
||||
} else {
|
||||
convertedPage.delete()
|
||||
}
|
||||
}
|
||||
|
||||
private fun getConversionWriter(
|
||||
targetMime: String,
|
||||
compressionLevel: Double?,
|
||||
): Pair<ImageWriter, ImageWriteParam>? {
|
||||
val writers = ImageIO.getImageWritersByMIMEType(targetMime)
|
||||
val writer =
|
||||
try {
|
||||
writers.next()
|
||||
} catch (_: NoSuchElementException) {
|
||||
return null
|
||||
}
|
||||
|
||||
val writerParams = writer.defaultWriteParam
|
||||
compressionLevel?.let {
|
||||
writerParams.compressionMode = ImageWriteParam.MODE_EXPLICIT
|
||||
writerParams.compressionQuality = it.toFloat()
|
||||
}
|
||||
|
||||
return writer to writerParams
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,10 +44,11 @@ class ThumbnailFileProvider(
|
||||
return true
|
||||
}
|
||||
|
||||
Manga.fetchMangaThumbnail(mangaId).first.use { image ->
|
||||
val (inputStream, mime) = Manga.fetchMangaThumbnail(mangaId)
|
||||
inputStream.use { image ->
|
||||
makeSureDownloadDirExists()
|
||||
val filePath = getThumbnailDownloadPath(mangaId)
|
||||
ImageResponse.saveImage(filePath, image)
|
||||
ImageResponse.saveImage(filePath, image, mime)
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.util.storage
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import libcore.net.MimeUtils
|
||||
import okhttp3.Response
|
||||
import okhttp3.internal.closeQuietly
|
||||
import java.io.File
|
||||
@@ -69,7 +70,12 @@ object ImageResponse {
|
||||
|
||||
try {
|
||||
if (response.code == 200) {
|
||||
val (actualSavePath, imageType) = saveImage(filePath, response.body.byteStream())
|
||||
val (actualSavePath, imageType) =
|
||||
saveImage(
|
||||
filePath,
|
||||
response.body.byteStream(),
|
||||
response.header("Content-Type"),
|
||||
)
|
||||
return pathToInputStream(actualSavePath) to imageType
|
||||
} else {
|
||||
throw Exception("request error! ${response.code}")
|
||||
@@ -87,20 +93,28 @@ object ImageResponse {
|
||||
fun saveImage(
|
||||
filePath: String,
|
||||
image: InputStream,
|
||||
mimeType: String?,
|
||||
): Pair<String, String> {
|
||||
val mimeType = mimeType?.takeIf { it.startsWith("image/") }?.lowercase()
|
||||
val tmpSavePath = "$filePath.tmp"
|
||||
val tmpSaveFile = File(tmpSavePath)
|
||||
image.use { input -> tmpSaveFile.outputStream().use { output -> input.copyTo(output) } }
|
||||
|
||||
// find image type
|
||||
val imageType =
|
||||
ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime
|
||||
?: "image/jpeg"
|
||||
ImageUtil.findImageType { tmpSaveFile.inputStream() }
|
||||
?: ImageUtil.ImageType.entries.find {
|
||||
it.mime == mimeType
|
||||
}
|
||||
val extension =
|
||||
imageType?.extension ?: mimeType?.let {
|
||||
MimeUtils.guessExtensionFromMimeType(it)
|
||||
} ?: "jpg"
|
||||
|
||||
val actualSavePath = "$filePath.${imageType.substringAfter("/")}"
|
||||
val actualSavePath = "$filePath.$extension"
|
||||
|
||||
tmpSaveFile.renameTo(File(actualSavePath))
|
||||
return Pair(actualSavePath, imageType)
|
||||
return Pair(actualSavePath, imageType?.mime ?: mimeType ?: "image/jpeg")
|
||||
}
|
||||
|
||||
fun clearCachedImage(
|
||||
|
||||
@@ -1,35 +1,28 @@
|
||||
package suwayomi.tachidesk.util
|
||||
|
||||
import eu.kanade.tachiyomi.network.NetworkHelper
|
||||
import eu.kanade.tachiyomi.network.POST
|
||||
import eu.kanade.tachiyomi.network.await
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import libcore.net.MimeUtils
|
||||
import okhttp3.Headers
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.MultipartBody
|
||||
import okhttp3.RequestBody.Companion.asRequestBody
|
||||
import suwayomi.tachidesk.graphql.types.DownloadConversion
|
||||
import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
|
||||
import uy.kohesive.injekt.injectLazy
|
||||
import java.awt.image.BufferedImage
|
||||
import java.io.File
|
||||
import java.io.InputStream
|
||||
import java.nio.file.Files
|
||||
import javax.imageio.ImageIO
|
||||
import kotlin.getValue
|
||||
|
||||
object ConversionUtil {
|
||||
val logger = KotlinLogging.logger {}
|
||||
|
||||
public fun readImage(image: File): BufferedImage? {
|
||||
val readers = ImageIO.getImageReadersBySuffix(image.extension)
|
||||
image.inputStream().use {
|
||||
ImageIO.createImageInputStream(it).use { inputStream ->
|
||||
for (reader in readers) {
|
||||
try {
|
||||
reader.setInput(inputStream)
|
||||
return reader.read(0)
|
||||
} catch (e: Throwable) {
|
||||
logger.debug(e) { "Reader ${reader.javaClass.name} not suitable" }
|
||||
} finally {
|
||||
reader.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info { "No suitable image converter found for ${image.name}" }
|
||||
return null
|
||||
}
|
||||
|
||||
public fun readImage(
|
||||
fun readImage(
|
||||
image: InputStream,
|
||||
mimeType: String,
|
||||
): BufferedImage? {
|
||||
@@ -49,4 +42,100 @@ object ConversionUtil {
|
||||
logger.info { "No suitable image converter found for $mimeType" }
|
||||
return null
|
||||
}
|
||||
|
||||
private val networkService: NetworkHelper by injectLazy()
|
||||
|
||||
/**
|
||||
* Send image to external HTTP service for post-processing
|
||||
* Returns the processed image stream or null if failed
|
||||
*/
|
||||
suspend fun imageHttpPostProcess(
|
||||
imageFile: File,
|
||||
conversion: DownloadConversion,
|
||||
mimeType: String,
|
||||
): InputStream? =
|
||||
try {
|
||||
logger.debug { "Sending ${imageFile.name} to HTTP converter: ${conversion.target}" }
|
||||
|
||||
val requestBody =
|
||||
MultipartBody
|
||||
.Builder()
|
||||
.setType(MultipartBody.FORM)
|
||||
.addFormDataPart(
|
||||
"image",
|
||||
imageFile.name,
|
||||
imageFile.asRequestBody(mimeType.toMediaType()),
|
||||
).build()
|
||||
|
||||
val client =
|
||||
networkService.client
|
||||
.newBuilder()
|
||||
.apply {
|
||||
if (conversion.callTimeout != null) {
|
||||
callTimeout(conversion.callTimeout!!)
|
||||
}
|
||||
if (conversion.connectTimeout != null) {
|
||||
connectTimeout(conversion.connectTimeout!!)
|
||||
}
|
||||
}.build()
|
||||
|
||||
val response =
|
||||
client
|
||||
.newCall(
|
||||
POST(
|
||||
conversion.target,
|
||||
body = requestBody,
|
||||
headers =
|
||||
Headers
|
||||
.Builder()
|
||||
.apply {
|
||||
conversion.headers?.forEach {
|
||||
set(it.key, it.value)
|
||||
}
|
||||
}.build(),
|
||||
),
|
||||
).await()
|
||||
logger.debug { "HTTP conversion successful for ${imageFile.name}" }
|
||||
response.body.byteStream()
|
||||
} catch (e: Exception) {
|
||||
logger.warn(e) { "HTTP conversion failed for ${imageFile.name}" }
|
||||
null
|
||||
}
|
||||
|
||||
/**
|
||||
* Overload that takes InputStream and mimeType, creates temp file for HTTP upload
|
||||
*/
|
||||
suspend fun imageHttpPostProcess(
|
||||
inputStream: InputStream,
|
||||
mimeType: String,
|
||||
conversion: DownloadConversion,
|
||||
): InputStream? =
|
||||
try {
|
||||
// Create temporary file from input stream
|
||||
val extension =
|
||||
MimeUtils.guessExtensionFromMimeType(mimeType)
|
||||
?: mimeType.substringAfter('/')
|
||||
|
||||
val tempFile = Files.createTempFile("conversion", ".$extension").toFile()
|
||||
tempFile.outputStream().use { output ->
|
||||
inputStream.copyTo(output)
|
||||
}
|
||||
|
||||
// Convert using file method
|
||||
val result = imageHttpPostProcess(tempFile, conversion, mimeType)
|
||||
|
||||
// Clean up temp file
|
||||
tempFile.delete()
|
||||
|
||||
result
|
||||
} catch (e: Exception) {
|
||||
logger.warn(e) { "Failed to create temp file for HTTP converter" }
|
||||
null
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a DownloadConversion target is an HTTP URL
|
||||
*/
|
||||
fun isHttpPostProcess(conversion: DownloadConversion): Boolean =
|
||||
conversion.target.startsWith("http://") || conversion.target.startsWith("https://")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user