fix #81 site search improved

gradle.properties

@@ -34,7 +34,7 @@ kotlin.native.cacheKind.linuxX64=none
 # On this environment, the system JDK 21 installation lacks `jlink`, causing
 # :lynglib:androidJdkImage to fail. Point Gradle to a JDK that includes `jlink`.
 # This affects only the JDK Gradle runs with; Kotlin/JVM target remains compatible.
-org.gradle.java.home=/home/sergeych/.jdks/corretto-21.0.9
+#org.gradle.java.home=/home/sergeych/.jdks/corretto-21.0.9
 android.experimental.lint.migrateToK2=false
 android.lint.useK2Uast=false
 kotlin.mpp.applyDefaultHierarchyTemplate=true

site/src/jsMain/kotlin/App.kt

@@ -106,6 +106,7 @@ fun App() {
                 when {
                     route.isBlank() -> HomePage()
                     route == "tryling" -> TryLyngPage()
+                    route.startsWith("search") -> SearchPage(route)
                     !isDocsRoute -> ReferencePage()
                     else -> DocsPage(
                         route = route,

site/src/jsMain/kotlin/Main.kt

@@ -449,13 +449,42 @@ private fun closeNavbarCollapse() {
 
 // ---------------- Site-wide search (client-side) ----------------
 
-private data class DocRecord(val path: String, val title: String, val text: String)
+internal data class DocRecord(val path: String, val title: String, val text: String)
 
 private var searchIndex: List<DocRecord>? = null
 private var searchBuilding = false
 private var searchInitDone = false
 
-private fun norm(s: String): String = s.lowercase()
+// ---- Search history (last 7 entries) ----
+private const val SEARCH_HISTORY_KEY = "lyng.search.history"
+
+private fun loadSearchHistory(): MutableList<String> {
+    return try {
+        val raw = window.localStorage.getItem(SEARCH_HISTORY_KEY) ?: return mutableListOf()
+        // Stored as newline-separated values to avoid JSON pitfalls across browsers
+        raw.split('\n').mapNotNull { it.trim() }.filter { it.isNotEmpty() }.toMutableList()
+    } catch (_: Throwable) {
+        mutableListOf()
+    }
+}
+
+private fun saveSearchHistory(list: List<String>) {
+    try {
+        window.localStorage.setItem(SEARCH_HISTORY_KEY, list.joinToString("\n"))
+    } catch (_: Throwable) { }
+}
+
+internal fun rememberSearchQuery(q: String) {
+    val query = q.trim()
+    if (query.isBlank()) return
+    val list = loadSearchHistory()
+    list.removeAll { it.equals(query, ignoreCase = true) }
+    list.add(0, query)
+    while (list.size > 7) list.removeAt(list.lastIndex)
+    saveSearchHistory(list)
+}
+
+internal fun norm(s: String): String = s.lowercase()
     .replace("`", " ")
     .replace("*", " ")
     .replace("#", " ")
@@ -466,7 +495,7 @@ private fun norm(s: String): String = s.lowercase()
     .replace(Regex("\\n+"), " ")
     .replace(Regex("\\s+"), " ").trim()
 
-private fun plainFromMarkdown(md: String): String {
+internal fun plainFromMarkdown(md: String): String {
     // Construct Regex instances at call time inside try/catch to avoid module init crashes
     // in browsers that are strict about Unicode RegExp parsing (Safari/Chrome).
     // Use non-greedy dot-all equivalents ("[\n\r\s\S]") instead of character classes with ']' where possible.
@@ -597,60 +626,24 @@ private fun scoreQuery(q: String, rec: DocRecord): Int {
     return score
 }
 
-private fun renderSearchResults(input: HTMLInputElement, menu: HTMLDivElement, q: String, results: List<DocRecord>) {
-    dlog("search-ui", "renderSearchResults q='$q' results=${results.size} building=$searchBuilding")
-    if (q.isBlank()) {
-        dlog("search-ui", "blank query -> hide menu")
-        menu.classList.remove("show")
-        menu.innerHTML = ""
-        return
-    }
-    if (results.isEmpty()) {
-        // If index is building, show a transient indexing indicator instead of hiding everything
-        if (searchBuilding) {
-            dlog("search-ui", "indexing in progress -> show placeholder")
-            menu.innerHTML = "<div class=\"dropdown-item disabled\">Indexing documentation…</div>"
-            menu.classList.add("show")
-        } else {
-            dlog("search-ui", "no results -> show 'No results' item")
-            val safeQ = q.replace("<", "&lt;").replace(">", "&gt;")
-            menu.innerHTML = "<div class=\"dropdown-item disabled\">No results for ‘$safeQ’</div>"
-            menu.classList.add("show")
-        }
-        return
-    }
-    val top = results.sortedByDescending { scoreQuery(q, it) }.take(8)
-    val items = buildString {
-        top.forEach { rec ->
-            append("<a href=\"#/${rec.path}\" class=\"dropdown-item\" data-path=\"${rec.path}\">")
-            append("<strong>")
-            append(rec.title)
-            append("</strong><br><small class=\"text-muted\">")
-            append(rec.path.substringAfter("docs/"))
-            append("</small></a>")
-        }
-    }
-    menu.innerHTML = items
-    // Position and show
-    menu.classList.add("show")
-    // Attach click handlers to enforce SPA navigation
-    val children = menu.getElementsByClassName("dropdown-item")
-    for (i in 0 until children.length) {
-        val a = children.item(i) as? HTMLAnchorElement ?: continue
-        a.onclick = { ev ->
-            ev.preventDefault()
-            val path = a.getAttribute("data-path")
-            if (path != null) {
-                val inputEl = input
-                val q = inputEl.value
-                val suffix = if (q.isNotBlank()) "?q=" + encodeURIComponent(q) else ""
-                dlog("nav", "search click -> navigate #/$path$suffix")
-                window.location.hash = "#/$path$suffix"
-                menu.classList.remove("show")
-                closeNavbarCollapse()
+private fun renderSearchHistoryDropdown(menu: HTMLDivElement) {
+    val hist = loadSearchHistory()
+    if (hist.isEmpty()) {
+        menu.innerHTML = "<div class=\"dropdown-item disabled\">No recent searches</div>"
+    } else {
+        val items = buildString {
+            hist.take(7).forEach { hq ->
+                val safe = hq.replace("<", "&lt;").replace(">", "&gt;")
+                append("<a href=\"#/search?q=" + encodeURIComponent(hq) + "\" class=\"dropdown-item\" data-q=\"")
+                append(safe)
+                append("\">")
+                append(safe)
+                append("</a>")
             }
         }
+        menu.innerHTML = items
     }
+    menu.classList.add("show")
 }
 
 private fun hideSearchResults(menu: HTMLDivElement) {
@@ -709,24 +702,9 @@ fun initTopSearch(attempt: Int = 0) {
     dlog("init", "initTopSearch: wiring handlers")
     val scope = MainScopeProvider.scope
 
-    // Debounced search runner
+    // Debounced dropdown history refresher
     val runSearch = debounce(scope, 120L) {
-        val q = input.value
-        dlog("search", "debounced runSearch execute q='$q'")
-        val results = performSearch(q)
-        renderSearchResults(input, menu, q, results)
-        // Also update highlights on the currently visible page content
-        try {
-            val root = document.querySelector(".markdown-body") as? HTMLElement
-            if (root != null) {
-                val terms = q.trim()
-                    .split(Regex("\\s+"))
-                    .map { it.lowercase() }
-                    .filter { it.isNotEmpty() }
-                val hits = highlightSearchHits(root, terms)
-                dlog("search", "live highlight updated, hits=$hits, terms=$terms")
-            }
-        } catch (_: Throwable) { }
+        renderSearchHistoryDropdown(menu)
     }
 
     // Keep the input focused when interacting with the dropdown so it doesn't blur/close
@@ -740,15 +718,8 @@ fun initTopSearch(attempt: Int = 0) {
         runSearch()
     }
     input.onfocus = {
-        // Proactively build the index on first focus for faster first results
-        dlog("event", "search onfocus")
-        scope.launch {
-            if (searchIndex == null && !searchBuilding) {
-                dlog("search", "onfocus -> buildSearchIndexOnce")
-                buildSearchIndexOnce()
-            }
-        }
-        runSearch()
+        dlog("event", "search onfocus: show history")
+        renderSearchHistoryDropdown(menu)
     }
     input.onkeydown = { ev ->
         val key = ev.asDynamic().key as String
@@ -758,28 +729,17 @@ fun initTopSearch(attempt: Int = 0) {
                 hideSearchResults(menu)
             }
             "Enter" -> {
-                // Navigate to the best match
-                scope.launch {
-                    val q = input.value
-                    // If index is building and results would be empty, wait for it once
-                    if (searchIndex == null || searchBuilding) {
-                        dlog("search", "Enter -> ensure index")
-                        buildSearchIndexOnce()
-                    }
-                    val results = performSearch(q)
-                    val best = results.maxByOrNull { scoreQuery(q, it) }
-                    if (best != null) {
-                        val suffix = if (q.isNotBlank()) "?q=" + encodeURIComponent(q) else ""
-                        dlog("nav", "Enter -> navigate #/${best.path}$suffix")
-                        window.location.hash = "#/${best.path}$suffix"
-                        hideSearchResults(menu)
-                        closeNavbarCollapse()
-                    } else {
-                        dlog("search", "Enter -> no results for q='$q'")
-                    }
+                val q = input.value.trim()
+                if (q.isNotBlank()) {
+                    rememberSearchQuery(q)
+                    val url = "#/search?q=" + encodeURIComponent(q)
+                    dlog("nav", "Enter -> navigate $url")
+                    window.location.hash = url
+                    hideSearchResults(menu)
+                    closeNavbarCollapse()
                 }
             }
-        }
+    }
     }
     // Hide on blur after a short delay to allow click
     input.onblur = {

site/src/jsMain/kotlin/SearchPage.kt

@@ -0,0 +1,176 @@
+/*
+ * Search results page: async, progressive site-wide search with relevance sorting.
+ */
+
+import androidx.compose.runtime.*
+import kotlinx.browser.window
+import kotlinx.coroutines.await
+import kotlinx.coroutines.delay
+import org.jetbrains.compose.web.attributes.*
+import org.jetbrains.compose.web.dom.*
+
+@Composable
+fun SearchPage(route: String) {
+    val terms = extractSearchTerms(route)
+    var qRaw by remember { mutableStateOf(terms.joinToString(" ")) }
+    var total by remember { mutableStateOf(0) }
+    var checked by remember { mutableStateOf(0) }
+    var done by remember { mutableStateOf(false) }
+    data class Item(val rec: DocRecord, val score: Int)
+    var items by remember { mutableStateOf(listOf<Item>()) }
+
+    LaunchedEffect(route) {
+        // Remember query in history
+        rememberSearchQuery(qRaw)
+        // Load index list
+        try {
+            checked = 0
+            done = false
+            items = emptyList()
+            val resp = window.fetch("docs-index.json").await()
+            if (!resp.ok) {
+                total = 0
+                done = true
+                return@LaunchedEffect
+            }
+            val arr = (kotlin.js.JSON.parse<dynamic>(resp.text().await()) as Array<String>).toList()
+            val list = arr.sorted()
+            total = list.size
+            // Process sequentially to keep UI responsive
+            for (path in list) {
+                try {
+                    val url = "./" + encodeURI(path)
+                    val r = window.fetch(url).await()
+                    if (!r.ok) {
+                        checked++
+                        continue
+                    }
+                    val body = r.text().await()
+                    if (body.contains("[//]: # (excludeFromIndex)")) {
+                        checked++
+                        continue
+                    }
+                    val title = extractTitleFromMarkdown(body) ?: path.substringAfterLast('/')
+                    val plain = plainFromMarkdown(body)
+                    val rec = DocRecord(path, title, plain)
+                    val score = scoreQueryAdvanced(terms, rec)
+                    if (score > 0) {
+                        val newList = (items + Item(rec, score)).sortedByDescending { it.score }.take(200)
+                        items = newList
+                    }
+                } catch (_: Throwable) { /* ignore one entry errors */ }
+                checked++
+                // Yield to UI
+                delay(0)
+            }
+        } finally {
+            done = true
+        }
+    }
+
+    H1 { Text("Search") }
+    if (terms.isEmpty()) {
+        P { Text("Type something in the search box above.") }
+        return
+    }
+    P {
+        Text("Query: ")
+        Code { Text(qRaw) }
+    }
+    if (!done) {
+        Div({ classes("d-flex", "align-items-center", "gap-2", "mb-3") }) {
+            Div({ classes("spinner-border", "spinner-border-sm") }) {}
+            Text("Searching… $checked / $total")
+        }
+    }
+    if (items.isEmpty() && done) {
+        P { Text("No results found.") }
+    } else {
+        Ul({ classes("list-unstyled") }) {
+            items.forEach { item ->
+                Li({ classes("mb-3") }) {
+                    val href = "#/${item.rec.path}?q=" + encodeURIComponent(qRaw)
+                    A(attrs = { attr("href", href) }) { Text(item.rec.title) }
+                    Br()
+                    Small({ classes("text-muted") }) { Text(item.rec.path.substringAfter("docs/")) }
+                }
+            }
+        }
+    }
+}
+
+// Advanced relevance scoring: combines coverage of distinct terms and proximity (minimal window).
+internal fun scoreQueryAdvanced(termsIn: List<String>, rec: DocRecord): Int {
+    val terms = termsIn.filter { it.isNotBlank() }.map { it.lowercase() }.distinct()
+    if (terms.isEmpty()) return 0
+    val title = norm(rec.title)
+    val text = rec.text
+
+    // Title coverage bonus
+    var titleCoverage = 0
+    for (t in terms) if (title.contains(t)) titleCoverage++
+
+    // Tokenize body and collect positions for each term by first prefix match
+    val wordRegex = Regex("[A-Za-z0-9_]+")
+    val posMap = HashMap<String, MutableList<Int>>()
+    var pos = 0
+    for (m in wordRegex.findAll(text)) {
+        val token = m.value.lowercase()
+        var matched: String? = null
+        var best = 0
+        for (t in terms) {
+            if (token.startsWith(t) && t.length > best) { best = t.length; matched = t }
+        }
+        if (matched != null) {
+            posMap.getOrPut(matched!!) { mutableListOf() }.add(pos)
+        }
+        pos++
+        if (pos > 100000) break // safety for extremely large docs
+    }
+    val coverage = terms.count { (posMap[it]?.isNotEmpty()) == true }
+    if (coverage == 0) return 0
+
+    // Proximity: minimal window length covering at least one occurrence of each covered term
+    var minWindow = Int.MAX_VALUE
+    if (coverage >= 2) {
+        // Prepare lists
+        val lists = terms.mapNotNull { t -> posMap[t]?.let { t to it } }.toList()
+        // K-way merge style window across sorted lists
+        val idx = IntArray(lists.size) { 0 }
+        // Initialize current positions
+        fun currentWindow(): Int {
+            var minPos = Int.MAX_VALUE
+            var maxPos = Int.MIN_VALUE
+            for (i in lists.indices) {
+                val p = lists[i].second[idx[i]]
+                if (p < minPos) minPos = p
+                if (p > maxPos) maxPos = p
+            }
+            return maxPos - minPos + 1
+        }
+        // Iterate advancing the pointer at the list with minimal current position
+        while (true) {
+            val win = currentWindow()
+            if (win < minWindow) minWindow = win
+            // Find list with minimal current pos
+            var minI = 0
+            var minVal = lists[0].second[idx[0]]
+            for (i in 1 until lists.size) {
+                val v = lists[i].second[idx[i]]
+                if (v < minVal) { minVal = v; minI = i }
+            }
+            idx[minI]++
+            if (idx[minI] >= lists[minI].second.size) break
+            // stop if window is already 1 (best possible)
+            if (minWindow <= 1) break
+        }
+        if (minWindow == Int.MAX_VALUE) minWindow = 100000
+    }
+
+    val coverageScore = coverage * 10000
+    val proximityScore = if (coverage >= 2) (5000.0 / (1 + minWindow)).toInt() else 0
+    val titleScore = titleCoverage * 3000
+    // Slight preference for shorter text
+    val lengthAdj = (200 - kotlin.math.min(200, text.length / 500))
+    return coverageScore + proximityScore + titleScore + lengthAdj
+}

site/src/jsTest/kotlin/SearchHistoryTest.kt

@@ -0,0 +1,31 @@
+import kotlinx.browser.window
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class SearchHistoryTest {
+    @BeforeTest
+    fun resetStorage() {
+        window.localStorage.removeItem("lyng.search.history")
+    }
+
+    @Test
+    fun keepsOnlySevenMostRecentUnique() {
+        // Add 9 queries
+        for (i in 1..9) rememberSearchQuery("query $i")
+        val raw = window.localStorage.getItem("lyng.search.history") ?: ""
+        val list = raw.split('\n').filter { it.isNotBlank() }
+        assertEquals(7, list.size, "history should contain 7 items")
+        assertEquals("query 9", list[0], "most recent should be first")
+        assertEquals("query 3", list.last(), "oldest retained should be 3")
+
+        // Re-insert existing should move to front without duplicates
+        rememberSearchQuery("query 5")
+        val raw2 = window.localStorage.getItem("lyng.search.history") ?: ""
+        val list2 = raw2.split('\n').filter { it.isNotBlank() }
+        assertEquals(7, list2.size)
+        assertEquals("query 5", list2[0])
+        assertTrue(list2.count { it == "query 5" } == 1)
+    }
+}

site/src/jsTest/kotlin/SearchScoringTest.kt

@@ -0,0 +1,29 @@
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class SearchScoringTest {
+    private fun rec(text: String, title: String = "Doc") = DocRecord("docs/a.md", title, norm(text))
+
+    @Test
+    fun zeroWhenNoTerms() {
+        assertEquals(0, scoreQueryAdvanced(emptyList(), rec("hello world")))
+    }
+
+    @Test
+    fun coverageMatters() {
+        val r = rec("alpha beta gamma alpha beta")
+        val s1 = scoreQueryAdvanced(listOf("alp"), r)
+        val s2 = scoreQueryAdvanced(listOf("alp", "bet"), r)
+        assertTrue(s2 > s1, "two-term coverage should score higher than one-term")
+    }
+
+    @Test
+    fun proximityImprovesScore() {
+        val near = rec("alpha beta gamma")
+        val far = rec(("alpha "+"x ").repeat(50) + "beta")
+        val sNear = scoreQueryAdvanced(listOf("alp", "bet"), near)
+        val sFar = scoreQueryAdvanced(listOf("alp", "bet"), far)
+        assertTrue(sNear > sFar, "closer terms should have higher score")
+    }
+}