From e25fc95cbfaced9572e360db321527885edb0685 Mon Sep 17 00:00:00 2001 From: sergeych Date: Sat, 6 Dec 2025 18:03:15 +0100 Subject: [PATCH] fix #81 site search improved --- gradle.properties | 2 +- site/src/jsMain/kotlin/App.kt | 1 + site/src/jsMain/kotlin/Main.kt | 160 +++++++----------- site/src/jsMain/kotlin/SearchPage.kt | 176 ++++++++++++++++++++ site/src/jsTest/kotlin/SearchHistoryTest.kt | 31 ++++ site/src/jsTest/kotlin/SearchScoringTest.kt | 29 ++++ 6 files changed, 298 insertions(+), 101 deletions(-) create mode 100644 site/src/jsMain/kotlin/SearchPage.kt create mode 100644 site/src/jsTest/kotlin/SearchHistoryTest.kt create mode 100644 site/src/jsTest/kotlin/SearchScoringTest.kt diff --git a/gradle.properties b/gradle.properties index ef520ed..235d2dd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -34,7 +34,7 @@ kotlin.native.cacheKind.linuxX64=none # On this environment, the system JDK 21 installation lacks `jlink`, causing # :lynglib:androidJdkImage to fail. Point Gradle to a JDK that includes `jlink`. # This affects only the JDK Gradle runs with; Kotlin/JVM target remains compatible. -org.gradle.java.home=/home/sergeych/.jdks/corretto-21.0.9 +#org.gradle.java.home=/home/sergeych/.jdks/corretto-21.0.9 android.experimental.lint.migrateToK2=false android.lint.useK2Uast=false kotlin.mpp.applyDefaultHierarchyTemplate=true \ No newline at end of file diff --git a/site/src/jsMain/kotlin/App.kt b/site/src/jsMain/kotlin/App.kt index 5cb23c3..a39ede9 100644 --- a/site/src/jsMain/kotlin/App.kt +++ b/site/src/jsMain/kotlin/App.kt @@ -106,6 +106,7 @@ fun App() { when { route.isBlank() -> HomePage() route == "tryling" -> TryLyngPage() + route.startsWith("search") -> SearchPage(route) !isDocsRoute -> ReferencePage() else -> DocsPage( route = route, diff --git a/site/src/jsMain/kotlin/Main.kt b/site/src/jsMain/kotlin/Main.kt index 231023c..6c95513 100644 --- a/site/src/jsMain/kotlin/Main.kt +++ b/site/src/jsMain/kotlin/Main.kt @@ -449,13 +449,42 @@ private fun closeNavbarCollapse() { // ---------------- Site-wide search (client-side) ---------------- -private data class DocRecord(val path: String, val title: String, val text: String) +internal data class DocRecord(val path: String, val title: String, val text: String) private var searchIndex: List? = null private var searchBuilding = false private var searchInitDone = false -private fun norm(s: String): String = s.lowercase() +// ---- Search history (last 7 entries) ---- +private const val SEARCH_HISTORY_KEY = "lyng.search.history" + +private fun loadSearchHistory(): MutableList { + return try { + val raw = window.localStorage.getItem(SEARCH_HISTORY_KEY) ?: return mutableListOf() + // Stored as newline-separated values to avoid JSON pitfalls across browsers + raw.split('\n').mapNotNull { it.trim() }.filter { it.isNotEmpty() }.toMutableList() + } catch (_: Throwable) { + mutableListOf() + } +} + +private fun saveSearchHistory(list: List) { + try { + window.localStorage.setItem(SEARCH_HISTORY_KEY, list.joinToString("\n")) + } catch (_: Throwable) { } +} + +internal fun rememberSearchQuery(q: String) { + val query = q.trim() + if (query.isBlank()) return + val list = loadSearchHistory() + list.removeAll { it.equals(query, ignoreCase = true) } + list.add(0, query) + while (list.size > 7) list.removeAt(list.lastIndex) + saveSearchHistory(list) +} + +internal fun norm(s: String): String = s.lowercase() .replace("`", " ") .replace("*", " ") .replace("#", " ") @@ -466,7 +495,7 @@ private fun norm(s: String): String = s.lowercase() .replace(Regex("\\n+"), " ") .replace(Regex("\\s+"), " ").trim() -private fun plainFromMarkdown(md: String): String { +internal fun plainFromMarkdown(md: String): String { // Construct Regex instances at call time inside try/catch to avoid module init crashes // in browsers that are strict about Unicode RegExp parsing (Safari/Chrome). // Use non-greedy dot-all equivalents ("[\n\r\s\S]") instead of character classes with ']' where possible. @@ -597,60 +626,24 @@ private fun scoreQuery(q: String, rec: DocRecord): Int { return score } -private fun renderSearchResults(input: HTMLInputElement, menu: HTMLDivElement, q: String, results: List) { - dlog("search-ui", "renderSearchResults q='$q' results=${results.size} building=$searchBuilding") - if (q.isBlank()) { - dlog("search-ui", "blank query -> hide menu") - menu.classList.remove("show") - menu.innerHTML = "" - return - } - if (results.isEmpty()) { - // If index is building, show a transient indexing indicator instead of hiding everything - if (searchBuilding) { - dlog("search-ui", "indexing in progress -> show placeholder") - menu.innerHTML = "
Indexing documentation…
" - menu.classList.add("show") - } else { - dlog("search-ui", "no results -> show 'No results' item") - val safeQ = q.replace("<", "<").replace(">", ">") - menu.innerHTML = "
No results for ‘$safeQ’
" - menu.classList.add("show") - } - return - } - val top = results.sortedByDescending { scoreQuery(q, it) }.take(8) - val items = buildString { - top.forEach { rec -> - append("") - append("") - append(rec.title) - append("
") - append(rec.path.substringAfter("docs/")) - append("") - } - } - menu.innerHTML = items - // Position and show - menu.classList.add("show") - // Attach click handlers to enforce SPA navigation - val children = menu.getElementsByClassName("dropdown-item") - for (i in 0 until children.length) { - val a = children.item(i) as? HTMLAnchorElement ?: continue - a.onclick = { ev -> - ev.preventDefault() - val path = a.getAttribute("data-path") - if (path != null) { - val inputEl = input - val q = inputEl.value - val suffix = if (q.isNotBlank()) "?q=" + encodeURIComponent(q) else "" - dlog("nav", "search click -> navigate #/$path$suffix") - window.location.hash = "#/$path$suffix" - menu.classList.remove("show") - closeNavbarCollapse() +private fun renderSearchHistoryDropdown(menu: HTMLDivElement) { + val hist = loadSearchHistory() + if (hist.isEmpty()) { + menu.innerHTML = "
No recent searches
" + } else { + val items = buildString { + hist.take(7).forEach { hq -> + val safe = hq.replace("<", "<").replace(">", ">") + append("") + append(safe) + append("") } } + menu.innerHTML = items } + menu.classList.add("show") } private fun hideSearchResults(menu: HTMLDivElement) { @@ -709,24 +702,9 @@ fun initTopSearch(attempt: Int = 0) { dlog("init", "initTopSearch: wiring handlers") val scope = MainScopeProvider.scope - // Debounced search runner + // Debounced dropdown history refresher val runSearch = debounce(scope, 120L) { - val q = input.value - dlog("search", "debounced runSearch execute q='$q'") - val results = performSearch(q) - renderSearchResults(input, menu, q, results) - // Also update highlights on the currently visible page content - try { - val root = document.querySelector(".markdown-body") as? HTMLElement - if (root != null) { - val terms = q.trim() - .split(Regex("\\s+")) - .map { it.lowercase() } - .filter { it.isNotEmpty() } - val hits = highlightSearchHits(root, terms) - dlog("search", "live highlight updated, hits=$hits, terms=$terms") - } - } catch (_: Throwable) { } + renderSearchHistoryDropdown(menu) } // Keep the input focused when interacting with the dropdown so it doesn't blur/close @@ -740,15 +718,8 @@ fun initTopSearch(attempt: Int = 0) { runSearch() } input.onfocus = { - // Proactively build the index on first focus for faster first results - dlog("event", "search onfocus") - scope.launch { - if (searchIndex == null && !searchBuilding) { - dlog("search", "onfocus -> buildSearchIndexOnce") - buildSearchIndexOnce() - } - } - runSearch() + dlog("event", "search onfocus: show history") + renderSearchHistoryDropdown(menu) } input.onkeydown = { ev -> val key = ev.asDynamic().key as String @@ -758,28 +729,17 @@ fun initTopSearch(attempt: Int = 0) { hideSearchResults(menu) } "Enter" -> { - // Navigate to the best match - scope.launch { - val q = input.value - // If index is building and results would be empty, wait for it once - if (searchIndex == null || searchBuilding) { - dlog("search", "Enter -> ensure index") - buildSearchIndexOnce() - } - val results = performSearch(q) - val best = results.maxByOrNull { scoreQuery(q, it) } - if (best != null) { - val suffix = if (q.isNotBlank()) "?q=" + encodeURIComponent(q) else "" - dlog("nav", "Enter -> navigate #/${best.path}$suffix") - window.location.hash = "#/${best.path}$suffix" - hideSearchResults(menu) - closeNavbarCollapse() - } else { - dlog("search", "Enter -> no results for q='$q'") - } + val q = input.value.trim() + if (q.isNotBlank()) { + rememberSearchQuery(q) + val url = "#/search?q=" + encodeURIComponent(q) + dlog("nav", "Enter -> navigate $url") + window.location.hash = url + hideSearchResults(menu) + closeNavbarCollapse() } } - } + } } // Hide on blur after a short delay to allow click input.onblur = { diff --git a/site/src/jsMain/kotlin/SearchPage.kt b/site/src/jsMain/kotlin/SearchPage.kt new file mode 100644 index 0000000..b52a0d0 --- /dev/null +++ b/site/src/jsMain/kotlin/SearchPage.kt @@ -0,0 +1,176 @@ +/* + * Search results page: async, progressive site-wide search with relevance sorting. + */ + +import androidx.compose.runtime.* +import kotlinx.browser.window +import kotlinx.coroutines.await +import kotlinx.coroutines.delay +import org.jetbrains.compose.web.attributes.* +import org.jetbrains.compose.web.dom.* + +@Composable +fun SearchPage(route: String) { + val terms = extractSearchTerms(route) + var qRaw by remember { mutableStateOf(terms.joinToString(" ")) } + var total by remember { mutableStateOf(0) } + var checked by remember { mutableStateOf(0) } + var done by remember { mutableStateOf(false) } + data class Item(val rec: DocRecord, val score: Int) + var items by remember { mutableStateOf(listOf()) } + + LaunchedEffect(route) { + // Remember query in history + rememberSearchQuery(qRaw) + // Load index list + try { + checked = 0 + done = false + items = emptyList() + val resp = window.fetch("docs-index.json").await() + if (!resp.ok) { + total = 0 + done = true + return@LaunchedEffect + } + val arr = (kotlin.js.JSON.parse(resp.text().await()) as Array).toList() + val list = arr.sorted() + total = list.size + // Process sequentially to keep UI responsive + for (path in list) { + try { + val url = "./" + encodeURI(path) + val r = window.fetch(url).await() + if (!r.ok) { + checked++ + continue + } + val body = r.text().await() + if (body.contains("[//]: # (excludeFromIndex)")) { + checked++ + continue + } + val title = extractTitleFromMarkdown(body) ?: path.substringAfterLast('/') + val plain = plainFromMarkdown(body) + val rec = DocRecord(path, title, plain) + val score = scoreQueryAdvanced(terms, rec) + if (score > 0) { + val newList = (items + Item(rec, score)).sortedByDescending { it.score }.take(200) + items = newList + } + } catch (_: Throwable) { /* ignore one entry errors */ } + checked++ + // Yield to UI + delay(0) + } + } finally { + done = true + } + } + + H1 { Text("Search") } + if (terms.isEmpty()) { + P { Text("Type something in the search box above.") } + return + } + P { + Text("Query: ") + Code { Text(qRaw) } + } + if (!done) { + Div({ classes("d-flex", "align-items-center", "gap-2", "mb-3") }) { + Div({ classes("spinner-border", "spinner-border-sm") }) {} + Text("Searching… $checked / $total") + } + } + if (items.isEmpty() && done) { + P { Text("No results found.") } + } else { + Ul({ classes("list-unstyled") }) { + items.forEach { item -> + Li({ classes("mb-3") }) { + val href = "#/${item.rec.path}?q=" + encodeURIComponent(qRaw) + A(attrs = { attr("href", href) }) { Text(item.rec.title) } + Br() + Small({ classes("text-muted") }) { Text(item.rec.path.substringAfter("docs/")) } + } + } + } + } +} + +// Advanced relevance scoring: combines coverage of distinct terms and proximity (minimal window). +internal fun scoreQueryAdvanced(termsIn: List, rec: DocRecord): Int { + val terms = termsIn.filter { it.isNotBlank() }.map { it.lowercase() }.distinct() + if (terms.isEmpty()) return 0 + val title = norm(rec.title) + val text = rec.text + + // Title coverage bonus + var titleCoverage = 0 + for (t in terms) if (title.contains(t)) titleCoverage++ + + // Tokenize body and collect positions for each term by first prefix match + val wordRegex = Regex("[A-Za-z0-9_]+") + val posMap = HashMap>() + var pos = 0 + for (m in wordRegex.findAll(text)) { + val token = m.value.lowercase() + var matched: String? = null + var best = 0 + for (t in terms) { + if (token.startsWith(t) && t.length > best) { best = t.length; matched = t } + } + if (matched != null) { + posMap.getOrPut(matched!!) { mutableListOf() }.add(pos) + } + pos++ + if (pos > 100000) break // safety for extremely large docs + } + val coverage = terms.count { (posMap[it]?.isNotEmpty()) == true } + if (coverage == 0) return 0 + + // Proximity: minimal window length covering at least one occurrence of each covered term + var minWindow = Int.MAX_VALUE + if (coverage >= 2) { + // Prepare lists + val lists = terms.mapNotNull { t -> posMap[t]?.let { t to it } }.toList() + // K-way merge style window across sorted lists + val idx = IntArray(lists.size) { 0 } + // Initialize current positions + fun currentWindow(): Int { + var minPos = Int.MAX_VALUE + var maxPos = Int.MIN_VALUE + for (i in lists.indices) { + val p = lists[i].second[idx[i]] + if (p < minPos) minPos = p + if (p > maxPos) maxPos = p + } + return maxPos - minPos + 1 + } + // Iterate advancing the pointer at the list with minimal current position + while (true) { + val win = currentWindow() + if (win < minWindow) minWindow = win + // Find list with minimal current pos + var minI = 0 + var minVal = lists[0].second[idx[0]] + for (i in 1 until lists.size) { + val v = lists[i].second[idx[i]] + if (v < minVal) { minVal = v; minI = i } + } + idx[minI]++ + if (idx[minI] >= lists[minI].second.size) break + // stop if window is already 1 (best possible) + if (minWindow <= 1) break + } + if (minWindow == Int.MAX_VALUE) minWindow = 100000 + } + + val coverageScore = coverage * 10000 + val proximityScore = if (coverage >= 2) (5000.0 / (1 + minWindow)).toInt() else 0 + val titleScore = titleCoverage * 3000 + // Slight preference for shorter text + val lengthAdj = (200 - kotlin.math.min(200, text.length / 500)) + return coverageScore + proximityScore + titleScore + lengthAdj +} diff --git a/site/src/jsTest/kotlin/SearchHistoryTest.kt b/site/src/jsTest/kotlin/SearchHistoryTest.kt new file mode 100644 index 0000000..5f9afdc --- /dev/null +++ b/site/src/jsTest/kotlin/SearchHistoryTest.kt @@ -0,0 +1,31 @@ +import kotlinx.browser.window +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SearchHistoryTest { + @BeforeTest + fun resetStorage() { + window.localStorage.removeItem("lyng.search.history") + } + + @Test + fun keepsOnlySevenMostRecentUnique() { + // Add 9 queries + for (i in 1..9) rememberSearchQuery("query $i") + val raw = window.localStorage.getItem("lyng.search.history") ?: "" + val list = raw.split('\n').filter { it.isNotBlank() } + assertEquals(7, list.size, "history should contain 7 items") + assertEquals("query 9", list[0], "most recent should be first") + assertEquals("query 3", list.last(), "oldest retained should be 3") + + // Re-insert existing should move to front without duplicates + rememberSearchQuery("query 5") + val raw2 = window.localStorage.getItem("lyng.search.history") ?: "" + val list2 = raw2.split('\n').filter { it.isNotBlank() } + assertEquals(7, list2.size) + assertEquals("query 5", list2[0]) + assertTrue(list2.count { it == "query 5" } == 1) + } +} diff --git a/site/src/jsTest/kotlin/SearchScoringTest.kt b/site/src/jsTest/kotlin/SearchScoringTest.kt new file mode 100644 index 0000000..d1218d3 --- /dev/null +++ b/site/src/jsTest/kotlin/SearchScoringTest.kt @@ -0,0 +1,29 @@ +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SearchScoringTest { + private fun rec(text: String, title: String = "Doc") = DocRecord("docs/a.md", title, norm(text)) + + @Test + fun zeroWhenNoTerms() { + assertEquals(0, scoreQueryAdvanced(emptyList(), rec("hello world"))) + } + + @Test + fun coverageMatters() { + val r = rec("alpha beta gamma alpha beta") + val s1 = scoreQueryAdvanced(listOf("alp"), r) + val s2 = scoreQueryAdvanced(listOf("alp", "bet"), r) + assertTrue(s2 > s1, "two-term coverage should score higher than one-term") + } + + @Test + fun proximityImprovesScore() { + val near = rec("alpha beta gamma") + val far = rec(("alpha "+"x ").repeat(50) + "beta") + val sNear = scoreQueryAdvanced(listOf("alp", "bet"), near) + val sFar = scoreQueryAdvanced(listOf("alp", "bet"), far) + assertTrue(sNear > sFar, "closer terms should have higher score") + } +}