SergeychWorks/toread
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

library improvements: search

Browse Source
This commit is contained in:
sergeych committed 2026-05-17 14:00:39 +03:00
1 parent ade4fb1896
commit 1239f15836
10 files changed
+618 -28
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

shared/src/commonMain/kotlin/net/sergeych/toread/storage/LibraryStorage.kt
+6
View File
@@ -30,6 +30,7 @@ data class BookRecord(
val language: String? = null,
val date: String? = null,
val description: String? = null,
val keywords: String? = null,
val coverImage: ByteArray? = null,
val coverImageMimeType: String? = null,
val createdAt: Long,
@@ -46,6 +47,7 @@ data class BookRecord(
language == other.language &&
date == other.date &&
description == other.description &&
keywords == other.keywords &&
coverImage.contentEquals(other.coverImage) &&
coverImageMimeType == other.coverImageMimeType &&
createdAt == other.createdAt &&
@@ -60,6 +62,7 @@ data class BookRecord(
result = 31 * result + (language?.hashCode() ?: 0)
result = 31 * result + (date?.hashCode() ?: 0)
result = 31 * result + (description?.hashCode() ?: 0)
result = 31 * result + (keywords?.hashCode() ?: 0)
result = 31 * result + (coverImage?.contentHashCode() ?: 0)
result = 31 * result + (coverImageMimeType?.hashCode() ?: 0)
result = 31 * result + createdAt.hashCode()
@@ -109,6 +112,7 @@ data class BookFileRecord(
val lastReadAt: Long? = null,
val createdAt: Long,
val updatedAt: Long,
val importedAt: Long = createdAt,
)
data class LibraryFileRecord(
@@ -125,6 +129,7 @@ data class LibraryFileRecord(
val lastSeenAt: Long? = null,
val readingStatus: BookReadingStatus = BookReadingStatus.NEW,
val lastReadAt: Long? = null,
val importedAt: Long,
)
data class ContentAnchor(
@@ -200,6 +205,7 @@ interface BookFileRepository {
fun findPrimaryDuplicateTarget(bodyClusterId: String?, bodyId: String?, rawSha256: String): BookFileRecord?
fun markDuplicateFiles(): Int
fun listLibraryFiles(limit: Int = 100, offset: Int = 0): List<LibraryFileRecord>
fun searchLibraryFiles(prefixes: List<String>, limit: Int = 100): List<LibraryFileRecord>
fun list(limit: Int = 500, offset: Int = 0): List<BookFileRecord>
fun listForBook(bookId: String): List<BookFileRecord>
fun updateReadingStatus(id: String, status: BookReadingStatus): Boolean
shared/src/jdbcMain/kotlin/net/sergeych/toread/storage/jdbc/H2LibraryDatabase.kt
+128 -12
View File
@@ -151,6 +151,7 @@ private fun migrate(connection: Connection) {
language VARCHAR,
published_date VARCHAR,
description CLOB,
keywords CLOB,
cover_image BLOB,
cover_image_mime_type VARCHAR,
created_at BIGINT NOT NULL,
@@ -160,6 +161,7 @@ private fun migrate(connection: Connection) {
)
statement.execute("ALTER TABLE books ADD COLUMN IF NOT EXISTS authors CLOB")
statement.execute("ALTER TABLE books ADD COLUMN IF NOT EXISTS published_date VARCHAR")
statement.execute("ALTER TABLE books ADD COLUMN IF NOT EXISTS keywords CLOB")
statement.execute("ALTER TABLE books ADD COLUMN IF NOT EXISTS cover_image BLOB")
statement.execute("ALTER TABLE books ADD COLUMN IF NOT EXISTS cover_image_mime_type VARCHAR")
statement.execute(
@@ -212,6 +214,7 @@ private fun migrate(connection: Connection) {
last_seen_at BIGINT,
reading_status VARCHAR NOT NULL DEFAULT 'NEW',
last_read_at BIGINT,
imported_at BIGINT NOT NULL,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
FOREIGN KEY (book_id) REFERENCES books(id),
@@ -223,6 +226,12 @@ private fun migrate(connection: Connection) {
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS duplicate_of_file_id VARCHAR")
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS reading_status VARCHAR NOT NULL DEFAULT 'NEW'")
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS last_read_at BIGINT")
statement.execute("ALTER TABLE book_files ADD COLUMN IF NOT EXISTS imported_at BIGINT")
connection.prepareStatement("UPDATE book_files SET imported_at = ? WHERE imported_at IS NULL").use { update ->
update.setLong(1, System.currentTimeMillis())
update.executeUpdate()
}
statement.execute("ALTER TABLE book_files ALTER COLUMN imported_at SET NOT NULL")
connection.createIndexIfMissing("idx_book_files_raw_sha256", "CREATE INDEX IF NOT EXISTS idx_book_files_raw_sha256 ON book_files(raw_sha256)")
connection.createIndexIfMissing("idx_book_files_book_id", "CREATE INDEX IF NOT EXISTS idx_book_files_book_id ON book_files(book_id)")
connection.createIndexIfMissing("idx_book_files_duplicate", "CREATE INDEX IF NOT EXISTS idx_book_files_duplicate ON book_files(duplicate_of_file_id)")
@@ -358,10 +367,10 @@ private class JdbcBookRepository(private val connection: Connection) : BookRepos
connection.prepareStatement(
"""
MERGE INTO books(
id, title, subtitle, authors, language, published_date, description,
id, title, subtitle, authors, language, published_date, description, keywords,
cover_image, cover_image_mime_type, created_at, updated_at
)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
statement.setString(1, book.id)
@@ -371,10 +380,11 @@ private class JdbcBookRepository(private val connection: Connection) : BookRepos
statement.setStringOrNull(5, book.language)
statement.setStringOrNull(6, book.date)
statement.setStringOrNull(7, book.description)
statement.setBytesOrNull(8, book.coverImage)
statement.setStringOrNull(9, book.coverImageMimeType)
statement.setLong(10, book.createdAt)
statement.setLong(11, book.updatedAt)
statement.setStringOrNull(8, book.keywords)
statement.setBytesOrNull(9, book.coverImage)
statement.setStringOrNull(10, book.coverImageMimeType)
statement.setLong(11, book.createdAt)
statement.setLong(12, book.updatedAt)
statement.executeUpdate()
}
}
@@ -477,9 +487,9 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
MERGE INTO book_files(
id, book_id, body_id, body_cluster_id, duplicate_of_file_id, raw_sha256, format,
mime_type, size_bytes, original_filename, storage_kind, storage_uri, content_object_id,
last_modified_millis, last_seen_at, reading_status, last_read_at, created_at, updated_at
last_modified_millis, last_seen_at, reading_status, last_read_at, imported_at, created_at, updated_at
)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
KEY(id) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
statement.setString(1, file.id)
@@ -499,8 +509,9 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
statement.setLongOrNull(15, file.lastSeenAt)
statement.setString(16, file.readingStatus.name)
statement.setLongOrNull(17, file.lastReadAt)
statement.setLong(18, file.createdAt)
statement.setLong(19, file.updatedAt)
statement.setLong(18, file.importedAt)
statement.setLong(19, file.createdAt)
statement.setLong(20, file.updatedAt)
statement.executeUpdate()
}
}
@@ -525,7 +536,8 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
f.storage_uri AS storage_uri,
f.last_seen_at AS last_seen_at,
f.reading_status AS reading_status,
f.last_read_at AS last_read_at
f.last_read_at AS last_read_at,
f.imported_at AS imported_at
FROM book_files f
LEFT JOIN books b ON b.id = f.book_id
WHERE f.id = ?
@@ -680,7 +692,8 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
f.storage_uri AS storage_uri,
f.last_seen_at AS last_seen_at,
f.reading_status AS reading_status,
f.last_read_at AS last_read_at
f.last_read_at AS last_read_at,
f.imported_at AS imported_at
FROM book_files f
LEFT JOIN books b ON b.id = f.book_id
WHERE f.duplicate_of_file_id IS NULL
@@ -698,6 +711,63 @@ private class JdbcBookFileRepository(private val connection: Connection) : BookF
}
}
override fun searchLibraryFiles(prefixes: List<String>, limit: Int): List<LibraryFileRecord> {
val normalizedPrefixes = prefixes.mapNotNull { it.trim().lowercase().takeIf(String::isNotBlank) }.distinct()
if (normalizedPrefixes.isEmpty()) return emptyList()
markDuplicateFiles()
val candidates = connection.prepareStatement(
"""
SELECT
f.id AS file_id,
f.book_id AS book_id,
b.title AS book_title,
b.authors AS book_authors,
b.language AS book_language,
b.published_date AS book_date,
b.description AS book_description,
b.keywords AS book_keywords,
f.format AS file_format,
f.size_bytes AS size_bytes,
f.original_filename AS original_filename,
f.storage_uri AS storage_uri,
f.last_seen_at AS last_seen_at,
f.reading_status AS reading_status,
f.last_read_at AS last_read_at,
f.imported_at AS imported_at
FROM book_files f
LEFT JOIN books b ON b.id = f.book_id
WHERE f.duplicate_of_file_id IS NULL
""".trimIndent()
).use { statement ->
statement.executeQuery().use { resultSet ->
resultSet.mapRows {
SearchCandidate(
file = it.toLibraryFileRecord(),
title = it.getString("book_title")?.takeIf(String::isNotBlank) ?: it.getString("original_filename"),
authors = it.getString("book_authors").fromDbList().joinToString(" "),
description = it.getString("book_description"),
keywords = it.getString("book_keywords"),
)
}
}
}
return candidates
.mapNotNull { candidate ->
candidate.searchRank(normalizedPrefixes)?.let { rank -> rank to candidate.file }
}
.sortedWith(
compareByDescending<Pair<SearchRank, LibraryFileRecord>> { it.first.hitCount }
.thenBy { it.first.hitSpan }
.thenBy { it.first.firstHit }
.thenBy { it.second.title?.lowercase().orEmpty() }
.thenBy { it.second.fileId }
)
.take(limit)
.map { it.second }
}
override fun list(limit: Int, offset: Int): List<BookFileRecord> {
return connection.prepareStatement("SELECT * FROM book_files ORDER BY updated_at DESC LIMIT ? OFFSET ?").use { statement ->
statement.setInt(1, limit)
@@ -905,12 +975,56 @@ private fun ResultSet.toBookRecord() = BookRecord(
language = getString("language"),
date = getString("published_date"),
description = getString("description"),
keywords = getString("keywords"),
coverImage = getBytes("cover_image"),
coverImageMimeType = getString("cover_image_mime_type"),
createdAt = getLong("created_at"),
updatedAt = getLong("updated_at"),
)
private data class SearchCandidate(
val file: LibraryFileRecord,
val title: String?,
val authors: String?,
val description: String?,
val keywords: String?,
)
private data class SearchRank(
val hitCount: Int,
val hitSpan: Int,
val firstHit: Int,
)
private fun SearchCandidate.searchRank(prefixes: List<String>): SearchRank? {
val hitPositions = buildList {
addAll(title.matchPositions(prefixes, 0))
addAll(authors.matchPositions(prefixes, 100_000))
addAll(keywords.matchPositions(prefixes, 200_000))
addAll(description.matchPositions(prefixes, 300_000))
}.sorted()
if (hitPositions.isEmpty()) return null
val first = hitPositions.first()
val last = hitPositions.last()
return SearchRank(
hitCount = hitPositions.size,
hitSpan = last - first,
firstHit = first,
)
}
private fun String?.matchPositions(prefixes: List<String>, fieldOffset: Int): List<Int> {
if (isNullOrBlank()) return emptyList()
return SearchWordRegex.findAll(lowercase())
.mapIndexedNotNull { index, match ->
if (prefixes.any { match.value.startsWith(it) }) fieldOffset + index else null
}
.toList()
}
private val SearchWordRegex = Regex("""[\p{L}\p{N}]+""")
private fun ResultSet.toLibraryFileRecord() = LibraryFileRecord(
fileId = getString("file_id"),
bookId = getString("book_id"),
@@ -925,6 +1039,7 @@ private fun ResultSet.toLibraryFileRecord() = LibraryFileRecord(
lastSeenAt = getLongOrNull("last_seen_at"),
readingStatus = getReadingStatus("reading_status"),
lastReadAt = getLongOrNull("last_read_at"),
importedAt = getLong("imported_at"),
)
private fun ResultSet.toBookBodyRecord() = BookBodyRecord(
@@ -961,6 +1076,7 @@ private fun ResultSet.toBookFileRecord() = BookFileRecord(
lastSeenAt = getLongOrNull("last_seen_at"),
readingStatus = getReadingStatus("reading_status"),
lastReadAt = getLongOrNull("last_read_at"),
importedAt = getLong("imported_at"),
createdAt = getLong("created_at"),
updatedAt = getLong("updated_at"),
)
shared/src/jdbcMain/kotlin/net/sergeych/toread/storage/jdbc/LibraryScanner.kt
+3
View File
@@ -104,6 +104,7 @@ class LibraryScanner(
storageUri = storageUri,
lastModifiedMillis = lastModifiedMillis,
lastSeenAt = now,
importedAt = now,
createdAt = now,
updatedAt = now,
)
@@ -134,6 +135,7 @@ class LibraryScanner(
language = book.language,
date = book.date,
description = book.annotation,
keywords = book.keywords,
coverImage = cover?.bytes,
coverImageMimeType = cover?.mimeType,
createdAt = now,
@@ -184,6 +186,7 @@ class LibraryScanner(
storageUri = storageUri,
lastModifiedMillis = lastModifiedMillis,
lastSeenAt = now,
importedAt = now,
createdAt = now,
updatedAt = now,
)
shared/src/jvmTest/kotlin/net/sergeych/toread/storage/jdbc/H2LibraryDatabaseTest.kt
+135
View File
@@ -16,6 +16,7 @@ import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
class H2LibraryDatabaseTest {
@Test
@@ -259,6 +260,89 @@ class H2LibraryDatabaseTest {
db.close()
}
@Test
fun searchesLibraryFilesByPrefixesAndRanksByHitCountThenDistance() {
val db = H2LibraryDatabase.openMemory("searchesLibraryFilesByPrefixesAndRanksByHitCountThenDistance")
val now = 1_700_000_000_000L
db.transaction {
books.upsert(
BookRecord(
id = "book-close",
title = "red blue alpha",
authors = listOf("Jane Example"),
createdAt = now,
updatedAt = now,
)
)
books.upsert(
BookRecord(
id = "book-far",
title = "red",
description = "alpha",
createdAt = now,
updatedAt = now,
)
)
books.upsert(
BookRecord(
id = "book-keyword",
title = "Unrelated",
keywords = "nebula archive",
createdAt = now,
updatedAt = now,
)
)
books.upsert(
BookRecord(
id = "book-ruso",
title = "El Ruso",
createdAt = now,
updatedAt = now,
)
)
listOf(
"file-close" to "book-close",
"file-far" to "book-far",
"file-keyword" to "book-keyword",
"file-ruso" to "book-ruso",
).forEachIndexed { index, (fileId, bookId) ->
files.upsert(
BookFileRecord(
id = fileId,
bookId = bookId,
rawSha256 = "sha-$fileId",
storageKind = BookFileStorageKind.EXTERNAL_URI,
createdAt = now + index,
updatedAt = now + index,
)
)
}
}
assertEquals(
listOf("file-close", "file-far"),
db.files.searchLibraryFiles(listOf("red", "alp")).map { it.fileId },
)
assertEquals(
listOf("file-close"),
db.files.searchLibraryFiles(listOf("blu")).map { it.fileId },
)
assertEquals(
listOf("file-keyword"),
db.files.searchLibraryFiles(listOf("neb")).map { it.fileId },
)
assertEquals(
listOf("file-ruso"),
db.files.searchLibraryFiles(listOf("rus")).map { it.fileId },
)
assertEquals(
emptyList(),
db.files.searchLibraryFiles(listOf("ous")).map { it.fileId },
)
db.close()
}
@Test
fun opensDatabaseWithExistingUppercaseIndex() {
val path = Files.createTempDirectory("toread-h2-index-").resolve("library").toString()
@@ -290,4 +374,55 @@ class H2LibraryDatabaseTest {
H2LibraryDatabase.openFile(path).close()
}
@Test
fun migratesLegacyBookFilesWithImportTime() {
val path = Files.createTempDirectory("toread-h2-imported-at-").resolve("library").toString()
Class.forName("org.h2.Driver")
DriverManager.getConnection("jdbc:h2:file:$path;DB_CLOSE_DELAY=0", "sa", "").use { connection ->
connection.createStatement().use { statement ->
statement.execute(
"""
CREATE TABLE book_files (
id VARCHAR PRIMARY KEY,
book_id VARCHAR,
body_id VARCHAR,
body_cluster_id VARCHAR,
duplicate_of_file_id VARCHAR,
raw_sha256 VARCHAR NOT NULL,
format VARCHAR,
mime_type VARCHAR,
size_bytes BIGINT,
original_filename VARCHAR,
storage_kind VARCHAR NOT NULL,
storage_uri VARCHAR,
content_object_id VARCHAR,
last_modified_millis BIGINT,
last_seen_at BIGINT,
reading_status VARCHAR NOT NULL DEFAULT 'NEW',
last_read_at BIGINT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)
""".trimIndent()
)
statement.execute(
"""
INSERT INTO book_files(
id, raw_sha256, storage_kind, reading_status, created_at, updated_at
) VALUES('legacy-file', 'legacy-sha', 'EXTERNAL_URI', 'NEW', 1, 2)
""".trimIndent()
)
}
}
val beforeMigration = System.currentTimeMillis()
val db = H2LibraryDatabase.openFile(path)
val file = assertNotNull(db.files.get("legacy-file"))
assertTrue(file.importedAt >= beforeMigration)
assertTrue(file.importedAt <= System.currentTimeMillis())
db.close()
}
}