SergeychWorks/toread
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

+image viewer, better dupes protection and background processing

Browse Source
This commit is contained in:
sergeych committed 2026-05-17 12:22:46 +03:00
1 parent 14c9863a83
commit ade4fb1896
18 files changed
+817 -127
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

shared/src/commonMain/kotlin/net/sergeych/toread/storage/LibraryStorage.kt
+5
View File
@@ -94,6 +94,7 @@ data class BookFileRecord(
val bookId: String? = null,
val bodyId: String? = null,
val bodyClusterId: String? = null,
val duplicateOfFileId: String? = null,
val rawSha256: String,
val format: String? = null,
val mimeType: String? = null,
@@ -194,6 +195,10 @@ interface BookFileRepository {
fun get(id: String): BookFileRecord?
fun getLibraryFile(id: String): LibraryFileRecord?
fun findByRawSha256(rawSha256: String): List<BookFileRecord>
fun findByOriginalFilenameSizeAndModified(originalFilename: String, sizeBytes: Long, lastModifiedMillis: Long): List<BookFileRecord>
fun findByOriginalFilenameSizeAndRawSha256(originalFilename: String, sizeBytes: Long, rawSha256: String): List<BookFileRecord>
fun findPrimaryDuplicateTarget(bodyClusterId: String?, bodyId: String?, rawSha256: String): BookFileRecord?
fun markDuplicateFiles(): Int
fun listLibraryFiles(limit: Int = 100, offset: Int = 0): List<LibraryFileRecord>
fun list(limit: Int = 500, offset: Int = 0): List<BookFileRecord>
fun listForBook(bookId: String): List<BookFileRecord>
shared/src/jdbcMain/kotlin/net/sergeych/toread/storage/jdbc/H2LibraryDatabase.kt
+146 -33
View File
@@ -199,6 +199,7 @@ private fun migrate(connection: Connection) {
book_id VARCHAR,
body_id VARCHAR,
body_cluster_id VARCHAR,
duplicate_of_file_id VARCHAR,
raw_sha256 VARCHAR NOT NULL,
format VARCHAR,
mime_type VARCHAR,
@@ -219,10 +220,14 @@ private fun migrate(connection: Connection) {
)
""".trimIndent()
)
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS duplicate_of_file_id VARCHAR")
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS reading_status VARCHAR NOT NULL DEFAULT 'NEW'")
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS last_read_at BIGINT")
connection.createIndexIfMissing("idx_book_files_raw_sha256", "CREATE INDEX IF NOT EXISTS idx_book_files_raw_sha256 ON book_files(raw_sha256)")
connection.createIndexIfMissing("idx_book_files_book_id", "CREATE INDEX IF NOT EXISTS idx_book_files_book_id ON book_files(book_id)")
connection.createIndexIfMissing("idx_book_files_duplicate", "CREATE INDEX IF NOT EXISTS idx_book_files_duplicate ON book_files(duplicate_of_file_id)")
connection.createIndexIfMissing("idx_book_files_name_size_mtime", "CREATE INDEX IF NOT EXISTS idx_book_files_name_size_mtime ON book_files(original_filename, size_bytes, last_modified_millis)")
connection.createIndexIfMissing("idx_book_files_name_size_hash", "CREATE INDEX IF NOT EXISTS idx_book_files_name_size_hash ON book_files(original_filename, size_bytes, raw_sha256)")
connection.createIndexIfMissing("idx_book_files_updated_at", "CREATE INDEX IF NOT EXISTS idx_book_files_updated_at ON book_files(updated_at)")
connection.createIndexIfMissing("idx_book_files_reading_order", "CREATE INDEX IF NOT EXISTS idx_book_files_reading_order ON book_files(reading_status, last_read_at)")
statement.execute(
@@ -470,31 +475,32 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
connection.prepareStatement(
"""
MERGE INTO book_files(
id, book_id, body_id, body_cluster_id, raw_sha256, format, mime_type, size_bytes,
original_filename, storage_kind, storage_uri, content_object_id, last_modified_millis,
last_seen_at, reading_status, last_read_at, created_at, updated_at
id, book_id, body_id, body_cluster_id, duplicate_of_file_id, raw_sha256, format,
mime_type, size_bytes, original_filename, storage_kind, storage_uri, content_object_id,
last_modified_millis, last_seen_at, reading_status, last_read_at, created_at, updated_at
)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
statement.setString(1, file.id)
statement.setStringOrNull(2, file.bookId)
statement.setStringOrNull(3, file.bodyId)
statement.setStringOrNull(4, file.bodyClusterId)
statement.setString(5, file.rawSha256)
statement.setStringOrNull(6, file.format)
statement.setStringOrNull(7, file.mimeType)
statement.setLongOrNull(8, file.sizeBytes)
statement.setStringOrNull(9, file.originalFilename)
statement.setString(10, file.storageKind.name)
statement.setStringOrNull(11, file.storageUri)
statement.setStringOrNull(12, file.contentObjectId)
statement.setLongOrNull(13, file.lastModifiedMillis)
statement.setLongOrNull(14, file.lastSeenAt)
statement.setString(15, file.readingStatus.name)
statement.setLongOrNull(16, file.lastReadAt)
statement.setLong(17, file.createdAt)
statement.setLong(18, file.updatedAt)
statement.setStringOrNull(5, file.duplicateOfFileId)
statement.setString(6, file.rawSha256)
statement.setStringOrNull(7, file.format)
statement.setStringOrNull(8, file.mimeType)
statement.setLongOrNull(9, file.sizeBytes)
statement.setStringOrNull(10, file.originalFilename)
statement.setString(11, file.storageKind.name)
statement.setStringOrNull(12, file.storageUri)
statement.setStringOrNull(13, file.contentObjectId)
statement.setLongOrNull(14, file.lastModifiedMillis)
statement.setLongOrNull(15, file.lastSeenAt)
statement.setString(16, file.readingStatus.name)
statement.setLongOrNull(17, file.lastReadAt)
statement.setLong(18, file.createdAt)
statement.setLong(19, file.updatedAt)
statement.executeUpdate()
}
}
@@ -534,27 +540,133 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
}
}
override fun listLibraryFiles(limit: Int, offset: Int): List<LibraryFileRecord> {
override fun findByOriginalFilenameSizeAndModified(
originalFilename: String,
sizeBytes: Long,
lastModifiedMillis: Long,
): List<BookFileRecord> {
return connection.prepareStatement(
"""
WITH visible_files AS (
SELECT * FROM book_files
WHERE original_filename = ?
AND size_bytes = ?
AND last_modified_millis = ?
ORDER BY created_at
""".trimIndent()
).use { statement ->
statement.setString(1, originalFilename)
statement.setLong(2, sizeBytes)
statement.setLong(3, lastModifiedMillis)
statement.executeQuery().use { resultSet -> resultSet.mapRows { it.toBookFileRecord() } }
}
}
override fun findByOriginalFilenameSizeAndRawSha256(
originalFilename: String,
sizeBytes: Long,
rawSha256: String,
): List<BookFileRecord> {
return connection.prepareStatement(
"""
SELECT * FROM book_files
WHERE original_filename = ?
AND size_bytes = ?
AND raw_sha256 = ?
ORDER BY created_at
""".trimIndent()
).use { statement ->
statement.setString(1, originalFilename)
statement.setLong(2, sizeBytes)
statement.setString(3, rawSha256)
statement.executeQuery().use { resultSet -> resultSet.mapRows { it.toBookFileRecord() } }
}
}
override fun findPrimaryDuplicateTarget(bodyClusterId: String?, bodyId: String?, rawSha256: String): BookFileRecord? {
return connection.prepareStatement(
"""
SELECT * FROM book_files
WHERE duplicate_of_file_id IS NULL
AND (
(? IS NOT NULL AND body_cluster_id = ?)
OR (? IS NOT NULL AND body_id = ?)
OR raw_sha256 = ?
)
ORDER BY
CASE
WHEN LOWER(COALESCE(format, '')) = 'fb2.zip'
OR LOWER(COALESCE(original_filename, '')) LIKE '%.fb2.zip'
THEN 0 ELSE 1
END,
CASE WHEN reading_status = 'READING' THEN 0 ELSE 1 END,
last_read_at DESC NULLS LAST,
updated_at DESC,
id
LIMIT 1
""".trimIndent()
).use { statement ->
statement.setStringOrNull(1, bodyClusterId)
statement.setStringOrNull(2, bodyClusterId)
statement.setStringOrNull(3, bodyId)
statement.setStringOrNull(4, bodyId)
statement.setString(5, rawSha256)
statement.executeQuery().use { resultSet ->
if (resultSet.next()) resultSet.toBookFileRecord() else null
}
}
}
override fun markDuplicateFiles(): Int {
val now = System.currentTimeMillis()
return connection.prepareStatement(
"""
MERGE INTO book_files(id, duplicate_of_file_id, updated_at)
KEY(id)
SELECT id, primary_file_id, ?
FROM (
SELECT
f.*,
ROW_NUMBER() OVER (
PARTITION BY COALESCE(f.body_cluster_id, f.body_id, f.book_id, f.raw_sha256, f.id)
id,
FIRST_VALUE(id) OVER (
PARTITION BY COALESCE(body_cluster_id, body_id, book_id, raw_sha256, id)
ORDER BY
CASE
WHEN LOWER(COALESCE(f.format, '')) = 'fb2.zip'
OR LOWER(COALESCE(f.original_filename, '')) LIKE '%.fb2.zip'
WHEN LOWER(COALESCE(format, '')) = 'fb2.zip'
OR LOWER(COALESCE(original_filename, '')) LIKE '%.fb2.zip'
THEN 0 ELSE 1
END,
CASE WHEN f.reading_status = 'READING' THEN 0 ELSE 1 END,
f.last_read_at DESC NULLS LAST,
f.updated_at DESC,
f.id
CASE WHEN reading_status = 'READING' THEN 0 ELSE 1 END,
last_read_at DESC NULLS LAST,
updated_at DESC,
id
) AS primary_file_id,
ROW_NUMBER() OVER (
PARTITION BY COALESCE(body_cluster_id, body_id, book_id, raw_sha256, id)
ORDER BY
CASE
WHEN LOWER(COALESCE(format, '')) = 'fb2.zip'
OR LOWER(COALESCE(original_filename, '')) LIKE '%.fb2.zip'
THEN 0 ELSE 1
END,
CASE WHEN reading_status = 'READING' THEN 0 ELSE 1 END,
last_read_at DESC NULLS LAST,
updated_at DESC,
id
) AS duplicate_rank
FROM book_files f
)
FROM book_files
WHERE duplicate_of_file_id IS NULL
) ranked
WHERE duplicate_rank > 1
""".trimIndent()
).use { statement ->
statement.setLong(1, now)
statement.executeUpdate()
}
}
override fun listLibraryFiles(limit: Int, offset: Int): List<LibraryFileRecord> {
markDuplicateFiles()
return connection.prepareStatement(
"""
SELECT
f.id AS file_id,
f.book_id AS book_id,
@@ -569,9 +681,9 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
f.last_seen_at AS last_seen_at,
f.reading_status AS reading_status,
f.last_read_at AS last_read_at
FROM visible_files f
FROM book_files f
LEFT JOIN books b ON b.id = f.book_id
WHERE f.duplicate_rank = 1
WHERE f.duplicate_of_file_id IS NULL
ORDER BY
CASE WHEN f.reading_status = 'READING' THEN 0 ELSE 1 END,
CASE WHEN f.reading_status = 'READING' THEN f.last_read_at END DESC NULLS LAST,
@@ -836,6 +948,7 @@ private fun ResultSet.toBookFileRecord() = BookFileRecord(
bookId = getString("book_id"),
bodyId = getString("body_id"),
bodyClusterId = getString("body_cluster_id"),
duplicateOfFileId = getString("duplicate_of_file_id"),
rawSha256 = getString("raw_sha256"),
format = getString("format"),
mimeType = getString("mime_type"),
shared/src/jdbcMain/kotlin/net/sergeych/toread/storage/jdbc/LibraryScanner.kt
+65 -20
View File
@@ -82,34 +82,65 @@ class LibraryScanner(
lastModifiedMillis: Long?,
): Boolean {
val rawSha256 = bytes.sha256Hex()
if (database.files.findByRawSha256(rawSha256).isNotEmpty()) return false
if (sizeBytes != null && database.files.findByOriginalFilenameSizeAndRawSha256(displayName, sizeBytes, rawSha256).isNotEmpty()) {
return false
}
val now = System.currentTimeMillis()
database.files.findPrimaryDuplicateTarget(bodyClusterId = null, bodyId = null, rawSha256 = rawSha256)?.let { duplicateTarget ->
database.files.upsert(
BookFileRecord(
id = "file-${UUID.randomUUID()}",
bookId = duplicateTarget.bookId,
bodyId = duplicateTarget.bodyId,
bodyClusterId = duplicateTarget.bodyClusterId,
duplicateOfFileId = duplicateTarget.id,
rawSha256 = rawSha256,
format = displayName.bookFormat(),
mimeType = displayName.bookMimeType(),
sizeBytes = sizeBytes,
originalFilename = displayName,
storageKind = BookFileStorageKind.EXTERNAL_URI,
storageUri = storageUri,
lastModifiedMillis = lastModifiedMillis,
lastSeenAt = now,
createdAt = now,
updatedAt = now,
)
)
return true
}
val book = Fb2Format.parse(bytes, displayName)
val cover = book.coverImage()
val canonicalText = book.canonicalText()
val bodyHash = canonicalText.encodeToByteArray().sha256Hex()
val now = System.currentTimeMillis()
val knownBody = database.bodies.findByExactTextHash(bodyHash, CanonicalizationVersion)
val bodyId = knownBody?.id ?: "body-${UUID.randomUUID()}"
val knownCluster = knownBody?.let { database.clusters.findByRepresentativeBodyId(it.id) }
val clusterId = knownCluster?.id ?: "cluster-${UUID.randomUUID()}"
val bookId = "book-${UUID.randomUUID()}"
val duplicateTarget = knownBody?.let {
database.files.findPrimaryDuplicateTarget(bodyClusterId = clusterId, bodyId = bodyId, rawSha256 = rawSha256)
}
val bookId = duplicateTarget?.bookId ?: "book-${UUID.randomUUID()}"
database.transaction {
books.upsert(
BookRecord(
id = bookId,
title = book.title.ifBlank { displayName.substringBeforeLast('.') },
authors = book.authors.mapNotNull { it.displayName.takeIf(String::isNotBlank) },
language = book.language,
date = book.date,
description = book.annotation,
coverImage = cover?.bytes,
coverImageMimeType = cover?.mimeType,
createdAt = now,
updatedAt = now,
if (duplicateTarget == null || duplicateTarget.bookId == null) {
books.upsert(
BookRecord(
id = bookId,
title = book.title.ifBlank { displayName.substringBeforeLast('.') },
authors = book.authors.mapNotNull { it.displayName.takeIf(String::isNotBlank) },
language = book.language,
date = book.date,
description = book.annotation,
coverImage = cover?.bytes,
coverImageMimeType = cover?.mimeType,
createdAt = now,
updatedAt = now,
)
)
)
}
if (knownBody == null) {
bodies.upsert(
BookBodyRecord(
@@ -143,6 +174,7 @@ class LibraryScanner(
bookId = bookId,
bodyId = bodyId,
bodyClusterId = clusterId,
duplicateOfFileId = duplicateTarget?.id,
rawSha256 = rawSha256,
format = displayName.bookFormat(),
mimeType = displayName.bookMimeType(),
@@ -160,14 +192,27 @@ class LibraryScanner(
return true
}
private fun importLinkedFile(file: File): Boolean =
importExternalFile(
private fun importLinkedFile(file: File): Boolean {
val sizeBytes = file.length()
val lastModifiedMillis = file.lastModified()
if (database.files.findByOriginalFilenameSizeAndModified(file.name, sizeBytes, lastModifiedMillis).isNotEmpty()) {
return false
}
return importExternalFile(
bytes = file.readBytes(),
displayName = file.name,
storageUri = file.absolutePath,
sizeBytes = file.length(),
lastModifiedMillis = file.lastModified(),
sizeBytes = sizeBytes,
lastModifiedMillis = lastModifiedMillis,
)
}
companion object {
fun countSupportedBookFiles(root: File): Int {
require(root.isDirectory) { "Scan root is not a directory: ${root.path}" }
return root.walkTopDown().count { it.isFile && it.isSupportedBookFile() }
}
}
}
private const val CanonicalizationVersion = 1
shared/src/jvmTest/kotlin/net/sergeych/toread/storage/jdbc/LibraryScannerTest.kt
+57
View File
@@ -1,9 +1,11 @@
package net.sergeych.toread.storage.jdbc
import java.io.File
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
class LibraryScannerTest {
@@ -35,6 +37,61 @@ class LibraryScannerTest {
}
}
@Test
fun scanMarksImportedDuplicateFilesImmediately() {
val root = findProjectRoot()
val source = File(root, "test_books/Maraini_Zapiski-Terezy-Numy.G7vc8A.872381.fb2.zip")
val tempDir = Files.createTempDirectory("yereed-dupe-scan").toFile()
try {
source.copyTo(File(tempDir, source.name))
source.copyTo(File(tempDir, "same-book-different-name.fb2.zip"))
val db = H2LibraryDatabase.openMemory("scanMarksImportedDuplicateFilesImmediately")
try {
val summary = LibraryScanner(db).scanSubtree(tempDir)
assertEquals(2, summary.scannedFiles)
assertEquals(2, summary.importedFiles)
assertEquals(0, summary.skippedFiles)
val files = db.files.list().sortedBy { it.duplicateOfFileId != null }
assertEquals(2, files.size)
assertNull(files.first().duplicateOfFileId)
assertEquals(files.first().id, files.last().duplicateOfFileId)
assertEquals(1, db.files.listLibraryFiles().size)
val second = LibraryScanner(db).scanSubtree(tempDir)
assertEquals(0, second.importedFiles)
assertEquals(2, second.skippedFiles)
} finally {
db.close()
}
} finally {
tempDir.deleteRecursively()
}
}
@Test
fun listingLibraryFilesPersistsLegacyDuplicateMarkers() {
val root = findProjectRoot()
val bytes = File(root, "test_books/Maraini_Zapiski-Terezy-Numy.G7vc8A.872381.fb2.zip").readBytes()
val db = H2LibraryDatabase.openMemory("listingLibraryFilesPersistsLegacyDuplicateMarkers")
try {
val scanner = LibraryScanner(db)
assertTrue(scanner.importExternalFile(bytes, "primary.fb2.zip", "/tmp/primary.fb2.zip", bytes.size.toLong(), 1L))
assertTrue(scanner.importExternalFile(bytes, "secondary.fb2.zip", "/tmp/secondary.fb2.zip", bytes.size.toLong(), 2L))
val duplicate = db.files.list().single { it.duplicateOfFileId != null }
db.files.upsert(duplicate.copy(duplicateOfFileId = null))
assertEquals(2, db.files.list().count { it.duplicateOfFileId == null })
assertEquals(1, db.files.listLibraryFiles().size)
assertEquals(1, db.files.list().count { it.duplicateOfFileId != null })
} finally {
db.close()
}
}
private fun findProjectRoot(): File {
var current = File(System.getProperty("user.dir")).absoluteFile
while (true) {