fix in site search and more tests on evalSession cancellation
This commit is contained in:
1 parent
fc01016a74
commit
a051280e0c
3 files changed
+84
-11
No files matched your search
@@ -512,8 +512,9 @@ internal fun plainFromMarkdown(md: String): String {
|
||||
// Use non-greedy dot-all equivalents ("[\n\r\s\S]") instead of character classes with ']' where possible.
|
||||
try {
|
||||
// Safer patterns (avoid unescaped ']' inside character classes):
|
||||
val reCodeBlocks = Regex("```[\\s\\S]*?```")
|
||||
val reInlineCode = Regex("`[^`]*`")
|
||||
val reBacktickCodeBlocks = Regex("```[\\s\\S]*?```")
|
||||
val reTildeCodeBlocks = Regex("~~~[\\s\\S]*?~~~")
|
||||
val reInlineCode = Regex("`([^`]*)`")
|
||||
val reBlockquote = Regex("^> +", setOf(RegexOption.MULTILINE))
|
||||
val reHeadings = Regex("^#+ +", setOf(RegexOption.MULTILINE))
|
||||
// Images:  — capture alt lazily with [\s\S]*? to avoid character class pitfalls
|
||||
@@ -523,9 +524,10 @@ internal fun plainFromMarkdown(md: String): String {
|
||||
|
||||
var t = md
|
||||
// Triple-backtick code blocks across lines
|
||||
t = t.replace(reCodeBlocks, " ")
|
||||
// Inline code
|
||||
t = t.replace(reInlineCode, " ")
|
||||
t = t.replace(reBacktickCodeBlocks, " ")
|
||||
t = t.replace(reTildeCodeBlocks, " ")
|
||||
// Keep inline code content so API names in prose and headings remain searchable
|
||||
t = t.replace(reInlineCode, "\$1")
|
||||
// Strip blockquotes and headings markers
|
||||
t = t.replace(reBlockquote, "")
|
||||
t = t.replace(reHeadings, "")
|
||||
@@ -539,7 +541,8 @@ internal fun plainFromMarkdown(md: String): String {
|
||||
// Minimal safe fallback: strip code blocks and inline code, then normalize
|
||||
var t = md
|
||||
t = t.replace(Regex("```[\\s\\S]*?```"), " ")
|
||||
t = t.replace(Regex("`[^`]*`"), " ")
|
||||
t = t.replace(Regex("~~~[\\s\\S]*?~~~"), " ")
|
||||
t = t.replace(Regex("`([^`]*)`"), "\$1")
|
||||
return norm(t)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,23 @@
|
||||
/*
|
||||
* Copyright 2026 Sergey S. Chernov real.sergeych@gmail.com
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class SearchScoringTest {
|
||||
@@ -26,4 +44,31 @@ class SearchScoringTest {
|
||||
val sFar = scoreQueryAdvanced(listOf("alp", "bet"), far)
|
||||
assertTrue(sNear > sFar, "closer terms should have higher score")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preservesInlineCodeTermsInHeadingsAndText() {
|
||||
val plain = plainFromMarkdown(
|
||||
"""
|
||||
### Preferred runtime: `EvalSession`
|
||||
|
||||
For host applications, prefer `EvalSession` as the main way to run scripts.
|
||||
""".trimIndent()
|
||||
)
|
||||
|
||||
assertTrue(plain.contains("evalsession"), "inline code terms should remain in searchable text")
|
||||
assertTrue(scoreQueryAdvanced(listOf("evalsession"), rec(plain)) > 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun stripsTildeCodeFencesLikeBacktickFences() {
|
||||
val plain = plainFromMarkdown(
|
||||
"""
|
||||
~~~kotlin
|
||||
val session = EvalSession()
|
||||
~~~
|
||||
""".trimIndent()
|
||||
)
|
||||
|
||||
assertFalse(plain.contains("evalsession"), "fenced code should not leak into the search corpus")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user