fixed highlighting and styles

docs/Iterator.md

@@ -9,6 +9,9 @@ To implement the iterator you need to implement only two abstract methods:
 
 ### hasNext(): Bool
 
+    // lets test
+    // offset
+
 Should return `true` if call to `next()` will return valid next element.
 
 ### next(): Obj

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt

@@ -84,12 +84,12 @@ private class Parser(fromPos: Pos) {
                 when (currentChar) {
                     '+' -> {
                         pos.advance()
-                        Token("+", from, Token.Type.PLUS2)
+                        Token("++", from, Token.Type.PLUS2)
                     }
 
                     '=' -> {
                         pos.advance()
-                        Token("+", from, Token.Type.PLUSASSIGN)
+                        Token("+=", from, Token.Type.PLUSASSIGN)
                     }
 
                     else ->
@@ -106,7 +106,7 @@ private class Parser(fromPos: Pos) {
 
                     '=' -> {
                         pos.advance()
-                        Token("-", from, Token.Type.MINUSASSIGN)
+                        Token("-=", from, Token.Type.MINUSASSIGN)
                     }
 
                     '>' -> {
@@ -129,17 +129,17 @@ private class Parser(fromPos: Pos) {
             '/' -> when (currentChar) {
                 '/' -> {
                     pos.advance()
-                    Token(loadToEndOfLine().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
+                    val body = loadToEndOfLine()
+                    // Include the leading '//' and do not trim; keep exact lexeme (excluding preceding codepoint)
+                    Token("//" + body, from, Token.Type.SINLGE_LINE_COMMENT)
                 }
 
                 '*' -> {
                     pos.advance()
-                    Token(
+                    val content = loadTo("*/")
-                        loadTo("*/")?.trim()
+                        ?: throw ScriptError(from, "Unterminated multiline comment")
-                            ?: throw ScriptError(from, "Unterminated multiline comment"),
+                    // loadTo consumes the closing fragment, so we are already after */
-                        from,
+                    Token("/*" + content + "*/", from, Token.Type.MULTILINE_COMMENT)
-                        Token.Type.MULTILINE_COMMENT
-                    )
                 }
 
                 '=' -> {
@@ -403,7 +403,8 @@ private class Parser(fromPos: Pos) {
             // could be integer, also hex:
             if (currentChar == 'x' && p1 == "0") {
                 pos.advance()
-                Token(loadChars { it in hexDigits }, start, Token.Type.HEX).also {
+                val hex = loadChars { it in hexDigits }
+                Token(hex, start, Token.Type.HEX).also {
                     if (currentChar.isLetter())
                         raise("invalid hex literal")
                 }
@@ -541,11 +542,11 @@ private class Parser(fromPos: Pos) {
 
     private fun loadToEndOfLine(): String {
         val result = StringBuilder()
-        val l = pos.line
+        // Read characters up to but not including the line break
-        do {
+        while (!pos.end && pos.currentChar != '\n') {
             result.append(pos.currentChar)
             pos.advance()
-        } while (pos.line == l)
+        }
         return result.toString()
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pos.kt

@@ -97,8 +97,5 @@ class MutablePos(private val from: Pos) {
     }
 
     override fun toString() = "($line:$column)"
-
+    
-    init {
-        if( lines[0].isEmpty()) advance()
-    }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Source.kt

@@ -19,9 +19,13 @@ package net.sergeych.lyng
 
 import net.sergeych.lyng.obj.ObjString
 
-class Source(val fileName: String, text: String) {
+class Source(val fileName: String, val text: String) {
 
-    val lines = text.lines().map { it.trimEnd() }
+    // Preserve original text characters exactly; do not trim trailing spaces.
+    // Split by "\n" boundaries as the lexer models line breaks uniformly as a single newline
+    // between logical lines, regardless of original platform line endings.
+    // We intentionally do NOT trim line ends to keep columns accurate.
+    val lines: List<String> = text.split('\n')
 
     val objSourceName by lazy { ObjString(fileName) }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/highlight/HighlightApi.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Cross-platform highlighting API for the Lyng language.
+ */
+package net.sergeych.lyng.highlight
+
+/** Represents a half-open character range [start, endExclusive). */
+data class TextRange(val start: Int, val endExclusive: Int) {
+    init { require(start <= endExclusive) { "Invalid range: $start..$endExclusive" } }
+}
+
+/** Kinds of tokens for syntax highlighting. */
+enum class HighlightKind {
+    Keyword,
+    TypeName,
+    Identifier,
+    Number,
+    String,
+    Char,
+    Regex,
+    Comment,
+    Operator,
+    Punctuation,
+    Label,
+    Directive,
+    Error,
+}
+
+/** A highlighted span: character range and its semantic/lexical kind. */
+data class HighlightSpan(val range: TextRange, val kind: HighlightKind)
+
+/** Base interface for Lyng syntax highlighters. */
+interface LyngHighlighter {
+    /**
+     * Produce highlight spans for the given [text]. Spans are non-overlapping and
+     * ordered by start position. Adjacent spans of the same kind may be merged.
+     */
+    fun highlight(text: String): List<HighlightSpan>
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/highlight/SimpleLyngHighlighter.kt

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.Token.Type
+import net.sergeych.lyng.parseLyng
+
+/** Extension that converts a [Pos] (line/column) into absolute character offset in the [Source] text. */
+fun Source.offsetOf(pos: Pos): Int {
+    var off = 0
+    // Sum full preceding lines + one '\n' per line (lines[] were created by String.lines())
+    var i = 0
+    while (i < pos.line) {
+        off += lines[i].length + 1 // assume \n as separator
+        i++
+    }
+    off += pos.column
+    return off
+}
+
+private val reservedIdKeywords = setOf("constructor", "property")
+// Fallback textual keywords that might come from the lexer as ID in some contexts (e.g., snippets)
+private val fallbackKeywordIds = setOf("and", "or", "not")
+
+/** Maps lexer token type (and sometimes value) to a [HighlightKind]. */
+private fun kindOf(type: Type, value: String): HighlightKind? = when (type) {
+    // identifiers and reserved ids
+    Type.ID -> when {
+        value in reservedIdKeywords -> HighlightKind.Keyword
+        value.lowercase() in fallbackKeywordIds -> HighlightKind.Keyword
+        else -> HighlightKind.Identifier
+    }
+
+    // numbers
+    Type.INT, Type.REAL, Type.HEX -> HighlightKind.Number
+
+    // text literals
+    Type.STRING, Type.STRING2 -> HighlightKind.String
+    Type.CHAR -> HighlightKind.Char
+    Type.REGEX -> HighlightKind.Regex
+
+    // comments
+    Type.SINLGE_LINE_COMMENT, Type.MULTILINE_COMMENT -> HighlightKind.Comment
+
+    // punctuation
+    Type.LPAREN, Type.RPAREN, Type.LBRACE, Type.RBRACE, Type.LBRACKET, Type.RBRACKET,
+    Type.COMMA, Type.SEMICOLON, Type.COLON -> HighlightKind.Punctuation
+
+    // textual control keywords
+    Type.IN, Type.NOTIN, Type.IS, Type.NOTIS, Type.AS, Type.ASNULL,
+    Type.AND, Type.OR, Type.NOT -> HighlightKind.Keyword
+
+    // labels / annotations
+    Type.LABEL, Type.ATLABEL -> HighlightKind.Label
+
+    // operators and symbolic constructs
+    Type.PLUS, Type.MINUS, Type.STAR, Type.SLASH, Type.PERCENT,
+    Type.ASSIGN, Type.PLUSASSIGN, Type.MINUSASSIGN, Type.STARASSIGN, Type.SLASHASSIGN, Type.PERCENTASSIGN,
+    Type.PLUS2, Type.MINUS2,
+    Type.EQ, Type.NEQ, Type.LT, Type.LTE, Type.GT, Type.GTE, Type.REF_EQ, Type.REF_NEQ, Type.MATCH, Type.NOTMATCH,
+    Type.DOT, Type.ARROW, Type.EQARROW, Type.QUESTION, Type.COLONCOLON,
+    Type.SHL, Type.SHR, Type.ELLIPSIS, Type.DOTDOT, Type.DOTDOTLT,
+    Type.NULL_COALESCE, Type.ELVIS, Type.NULL_COALESCE_INDEX, Type.NULL_COALESCE_INVOKE, Type.NULL_COALESCE_BLOCKINVOKE,
+    Type.SHUTTLE,
+    // bitwise textual operators (treat as operators for visuals)
+    Type.BITAND, Type.BITOR, Type.BITXOR, Type.BITNOT -> HighlightKind.Operator
+
+    // non-highlighting tokens
+    Type.NEWLINE, Type.EOF -> null
+}
+
+/** Merge contiguous spans of the same [HighlightKind] to reduce output size. */
+private fun mergeAdjacent(spans: List<HighlightSpan>): List<HighlightSpan> {
+    if (spans.isEmpty()) return spans
+    val out = ArrayList<HighlightSpan>(spans.size)
+    var prev = spans[0]
+    for (i in 1 until spans.size) {
+        val cur = spans[i]
+        if (cur.kind == prev.kind && cur.range.start == prev.range.endExclusive) {
+            prev = HighlightSpan(TextRange(prev.range.start, cur.range.endExclusive), prev.kind)
+        } else {
+            out += prev
+            prev = cur
+        }
+    }
+    out += prev
+    return out
+}
+
+/** Simple highlighter using the existing Lyng lexer (no incremental support yet). */
+class SimpleLyngHighlighter : LyngHighlighter {
+    override fun highlight(text: String): List<HighlightSpan> {
+        val src = Source("<snippet>", text)
+        val tokens = parseLyng(src)
+        val raw = ArrayList<HighlightSpan>(tokens.size)
+        fun adjustQuoteSpan(startOffset: Int, quoteChar: Char): TextRange {
+            var s = startOffset
+            if (s > 0 && text[s - 1] == quoteChar) s -= 1
+            var i = s + 1
+            while (i < text.length) {
+                val ch = text[i]
+                if (ch == '\\') {
+                    i += if (i + 1 < text.length) 2 else 1
+                    continue
+                }
+                if (ch == quoteChar) {
+                    return TextRange(s, i + 1)
+                }
+                i++
+            }
+            // Unterminated, highlight till end
+            return TextRange(s, text.length)
+        }
+
+        for (t in tokens) {
+            val k = kindOf(t.type, t.value) ?: continue
+            val start = src.offsetOf(t.pos)
+            val range = when (t.type) {
+                Type.STRING, Type.STRING2 -> adjustQuoteSpan(start, '"')
+                Type.CHAR -> adjustQuoteSpan(start, '\'')
+                else -> TextRange(start, (start + t.value.length).coerceAtMost(text.length))
+            }
+            if (range.endExclusive > range.start) raw += HighlightSpan(range, k)
+        }
+        // Adjust single-line comment spans to extend till EOL to compensate for lexer offset/length quirks
+        val adjusted = extendSingleLineCommentsToEol(text, raw)
+        // Spans are in order; merge adjacent of the same kind for compactness
+        return mergeAdjacent(adjusted)
+    }
+}
+
+/**
+ * Workaround/fix: ensure that single-line comment spans that start with `//` extend until the end of line.
+ * Drops any subsequent spans that would overlap the extended comment on the same line.
+ */
+private fun extendSingleLineCommentsToEol(
+    text: String,
+    spans: List<HighlightSpan>
+): List<HighlightSpan> {
+    if (spans.isEmpty()) return spans
+    val out = ArrayList<HighlightSpan>(spans.size)
+    var i = 0
+    while (i < spans.size) {
+        val s = spans[i]
+        if (s.kind == HighlightKind.Comment) {
+            // Check the original text actually has '//' at span start to avoid touching block comments
+            val start = s.range.start
+            val ahead = if (start in text.indices) text.substring(start, minOf(text.length, start + 2)) else ""
+            if (ahead == "//") {
+                // Extend to end of current line
+                val eol = text.indexOf('\n', start)
+                val newEnd = if (eol >= 0) eol else text.length
+                var j = i + 1
+                while (j < spans.size && spans[j].range.start < newEnd) {
+                    // Consume all overlapping spans on this line
+                    j++
+                }
+                out += HighlightSpan(TextRange(start, newEnd), s.kind)
+                i = j
+                continue
+            }
+        }
+        out += s
+        i++
+    }
+    return out
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/CommentEolTest.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class CommentEolTest {
+
+    @Test
+    fun singleLineCommentExtendsToEol() {
+        val line = "// see the difference: apply changes this to newly created Point:"
+        val text = "$line\nnext"
+
+        val spans = SimpleLyngHighlighter().highlight(text)
+        // Find the comment span
+        val cmt = spans.firstOrNull { it.kind == HighlightKind.Comment }
+        assertTrue(cmt != null, "Expected a comment span")
+        // It should start at 0 and extend exactly to the end of the line (before \n)
+        val eol = text.indexOf('\n')
+        assertEquals(0, cmt!!.range.start, "Comment should start at column 0")
+        assertEquals(eol, cmt.range.endExclusive, "Comment should extend to EOL")
+        // Ensure there is no other span overlapping within the same line
+        spans.filter { it !== cmt }.forEach {
+            assertTrue(it.range.start >= eol, "No span should start before EOL for single-line comment")
+        }
+    }
+
+    @Test
+    fun blockCommentNotExtendedPastClosing() {
+        val text = "/* block */ rest"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val cmt = spans.firstOrNull { it.kind == HighlightKind.Comment }
+        assertTrue(cmt != null, "Expected a block comment span")
+        // The comment should end right after "/* block */"
+        val expectedEnd = "/* block */".length
+        assertEquals(expectedEnd, cmt!!.range.endExclusive, "Block comment should not be extended to EOL")
+    }
+
+    @Test
+    fun twoSingleLineCommentsEachToTheirEol() {
+        val text = "// first\n// second\nend"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val cmts = spans.filter { it.kind == HighlightKind.Comment }
+        assertEquals(2, cmts.size, "Expected two single-line comment spans")
+
+        val eol1 = text.indexOf('\n')
+        assertEquals(0, cmts[0].range.start)
+        assertEquals(eol1, cmts[0].range.endExclusive)
+
+        val line2Start = eol1 + 1
+        val eol2 = text.indexOf('\n', line2Start)
+        assertEquals(line2Start, cmts[1].range.start)
+        assertEquals(eol2, cmts[1].range.endExclusive)
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/HighlightMappingTest.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class HighlightMappingTest {
+
+    private fun spansToLabeled(text: String, spans: List<HighlightSpan>): List<Pair<String, HighlightKind>> =
+        spans.map { text.substring(it.range.start, it.range.endExclusive) to it.kind }
+
+    @Test
+    fun keywordsAndIdentifiers() {
+        val text = "a and b or not c"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+
+        // Expect identifiers and keywords in order
+        assertTrue(labeled.any { it.first == "a" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "and" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "b" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "or" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "not" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "c" && it.second == HighlightKind.Identifier })
+    }
+
+    @Test
+    fun reservedWordsConstructorPropertyAsKeywords() {
+        val text = "constructor property foo"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "constructor" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "property" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "foo" && it.second == HighlightKind.Identifier })
+    }
+
+    @Test
+    fun numbersAndStringsAndChar() {
+        val text = "123 0xFF 1.0 'c' \"s\""
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "123" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first.lowercase() == "0xff" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "1.0" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "'c'" && it.second == HighlightKind.Char })
+        assertTrue(labeled.any { it.first == "\"s\"" && it.second == HighlightKind.String })
+    }
+
+    @Test
+    fun commentsHighlighted() {
+        val text = "// line\n/* block */"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first.startsWith("//") && it.second == HighlightKind.Comment })
+        assertTrue(labeled.any { it.first.startsWith("/*") && it.second == HighlightKind.Comment })
+    }
+
+    @Test
+    fun operatorsAndPunctuation() {
+        val text = "a+b; (x)"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "+" && it.second == HighlightKind.Operator })
+        assertTrue(labeled.any { it.first == ";" && it.second == HighlightKind.Punctuation })
+        assertTrue(labeled.any { it.first == "(" && it.second == HighlightKind.Punctuation })
+        assertTrue(labeled.any { it.first == ")" && it.second == HighlightKind.Punctuation })
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/RegressionAssertEqualsTest.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Regression tests for highlighting real-world snippet mentioned in the issue.
+ */
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class RegressionAssertEqualsTest {
+
+    private fun spansToLabeled(text: String, spans: List<HighlightSpan>): List<Pair<String, HighlightKind>> =
+        spans.map { text.substring(it.range.start, it.range.endExclusive) to it.kind }
+
+    @Test
+    fun assertEqualsSnippetIsTokenizedAndHighlightedSane() {
+        val text = "assertEquals( [9,10], r.takeLast(2).toList() )"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        // Debug print to help diagnose failures across targets
+        println("[DEBUG_LOG] labeled spans: " + labeled.joinToString(" | ") { "${it.first}:{${it.second}}" })
+
+        // Ensure identifier is not split: whole 'assertEquals' must be Identifier
+        assertTrue(labeled.any { it.first == "assertEquals" && it.second == HighlightKind.Identifier })
+
+        // Brackets and parentheses must be punctuation; spans may merge adjacent punctuation,
+        // so accept combined tokens like "()" or "],"
+        fun hasPunct(containing: Char) = labeled.any { containing in it.first && it.second == HighlightKind.Punctuation }
+        assertTrue(hasPunct('('))
+        assertTrue(hasPunct(')'))
+        assertTrue(hasPunct('['))
+        assertTrue(hasPunct(']'))
+        assertTrue(hasPunct(','))
+
+        // Numbers 9, 10 and 2 should be recognized as numbers
+        assertTrue(labeled.any { it.first == "9" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "10" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "2" && it.second == HighlightKind.Number })
+
+        // Method chain identifiers and dots/operators
+        assertTrue(labeled.any { it.first == "." && it.second == HighlightKind.Operator })
+        assertTrue(labeled.any { it.first == "r" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "takeLast" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "toList" && it.second == HighlightKind.Identifier })
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/SourceOffsetTest.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Verify that Source/Pos → absolute offset mapping is consistent and preserves
+ * exact characters, including trailing spaces and Windows CRLF endings.
+ */
+package net.sergeych.lyng.highlight
+
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Source
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class SourceOffsetTest {
+
+    @Test
+    fun preservesTrailingSpacesInColumns() {
+        val txt = "abc  \nxyz" // two trailing spaces before the newline
+        val src = Source("snippet", txt)
+        // line 0: "abc  " length 5
+        val p = Pos(src, 0, 4) // column at the last space
+        val off = src.offsetOf(p)
+        assertEquals(4, off)
+        // Take substring from start to this pos, it must include two spaces
+        assertEquals("abc ", txt.substring(0, off))
+    }
+
+    @Test
+    fun crlfLineEndingsDoNotBreakOffsets() {
+        val txt = "a\r\nb\r\nc" // three lines, split by \n; \r remain at line ends
+        val src = Source("snippet", txt)
+        // Position at start of line 2 ('c')
+        val p = Pos(src, 2, 0)
+        val off = src.offsetOf(p)
+        // Offsets: line0 len=2 ("a\r"), plus one for \n, line1 len=2 ("b\r"), plus one for \n => 2+1+2+1=6
+        assertEquals(6, off)
+        assertEquals('c', txt[off])
+    }
+}

site/build.gradle.kts

@@ -49,6 +49,8 @@ kotlin {
                 implementation("org.jetbrains.compose.html:html-core:1.9.3")
                 // Coroutines for JS (used for fetching docs)
                 implementation(libs.kotlinx.coroutines.core)
+                // Lyng highlighter (common, used from JS)
+                implementation(project(":lynglib"))
                 // Markdown parser (NPM)
                 implementation(npm("marked", "12.0.2"))
             }

site/src/jsMain/kotlin/HighlightSupport.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Thin site-side wrapper for highlighting text with lynglib and producing HTML spans
+ * with the same CSS classes as used in Main.kt.
+ */
+package net.sergeych.site
+
+import net.sergeych.lyng.highlight.HighlightKind
+import net.sergeych.lyng.highlight.SimpleLyngHighlighter
+
+object SiteHighlight {
+    private fun cssClassForKind(kind: HighlightKind): String = when (kind) {
+        HighlightKind.Keyword -> "hl-kw"
+        HighlightKind.TypeName -> "hl-ty"
+        HighlightKind.Identifier -> "hl-id"
+        HighlightKind.Number -> "hl-num"
+        HighlightKind.String -> "hl-str"
+        HighlightKind.Char -> "hl-ch"
+        HighlightKind.Regex -> "hl-rx"
+        HighlightKind.Comment -> "hl-cmt"
+        HighlightKind.Operator -> "hl-op"
+        HighlightKind.Punctuation -> "hl-punc"
+        HighlightKind.Label -> "hl-lbl"
+        HighlightKind.Directive -> "hl-dir"
+        HighlightKind.Error -> "hl-err"
+    }
+
+    private fun htmlEscape(s: String): String = buildString(s.length) {
+        for (ch in s) when (ch) {
+            '<' -> append("&lt;")
+            '>' -> append("&gt;")
+            '&' -> append("&amp;")
+            '"' -> append("&quot;")
+            '\'' -> append("&#39;")
+            else -> append(ch)
+        }
+    }
+
+    fun renderHtml(text: String): String {
+        val highlighter = SimpleLyngHighlighter()
+        val spans = highlighter.highlight(text)
+        if (spans.isEmpty()) return htmlEscape(text)
+        val sb = StringBuilder(text.length + spans.size * 16)
+        var pos = 0
+        for (s in spans) {
+            if (s.range.start > pos) sb.append(htmlEscape(text.substring(pos, s.range.start)))
+            val cls = cssClassForKind(s.kind)
+            sb.append('<').append("span class=\"").append(cls).append('\"').append('>')
+            sb.append(htmlEscape(text.substring(s.range.start, s.range.endExclusive)))
+            sb.append("</span>")
+            pos = s.range.endExclusive
+        }
+        if (pos < text.length) sb.append(htmlEscape(text.substring(pos)))
+        return sb.toString()
+    }
+}

site/src/jsMain/kotlin/Main.kt

@@ -38,7 +38,6 @@ fun App() {
     var toc by remember { mutableStateOf<List<TocItem>>(emptyList()) }
     var activeTocId by remember { mutableStateOf<String?>(null) }
     var contentEl by remember { mutableStateOf<HTMLElement?>(null) }
-    var theme by remember { mutableStateOf(detectInitialTheme()) }
     val isDocsRoute = route.startsWith("docs/")
     // A stable key for the current document path (without fragment). Used to avoid
     // re-fetching when only the in-page anchor changes.
@@ -203,24 +202,6 @@ fun App() {
                         onClick { it.preventDefault(); window.location.hash = "#/reference" }
                     }) { Text("Reference") }
 
-                    // Theme toggle
-                    Button(attrs = {
-                        classes("btn", "btn-sm", "btn-outline-secondary")
-                        onClick {
-                            theme = if (theme == Theme.Dark) Theme.Light else Theme.Dark
-                            applyTheme(theme)
-                            saveThemePreference(theme)
-                        }
-                    }) {
-                        if (theme == Theme.Dark) {
-                            I({ classes("bi", "bi-sun") })
-                            Text(" Light")
-                        } else {
-                            I({ classes("bi", "bi-moon") })
-                            Text(" Dark")
-                        }
-                    }
-
                     // Sample quick links
                     DocLink("Iterable.md")
                     DocLink("Iterator.md")
@@ -316,10 +297,12 @@ fun routeToPath(route: String): String {
 fun stripFragment(route: String): String = route.substringBefore('#')
 
 fun renderMarkdown(src: String): String =
-    ensureBootstrapCodeBlocks(
+    highlightLyngHtml(
-        ensureBootstrapTables(
+        ensureBootstrapCodeBlocks(
-            ensureDefinitionLists(
+            ensureBootstrapTables(
-                marked.parse(src)
+                ensureDefinitionLists(
+                    marked.parse(src)
+                )
             )
         )
     )
@@ -383,36 +366,15 @@ private fun ReferencePage() {
     }
 }
 
-// ---- Theme handling ----
+// ---- Theme handling: follow system theme automatically ----
 
-private enum class Theme { Light, Dark }
+private fun applyTheme(isDark: Boolean) {
-
-private fun detectInitialTheme(): Theme {
-    // Try user preference from localStorage
-    val stored = try { window.localStorage.getItem("theme") } catch (_: Throwable) { null }
-    if (stored == "dark") return Theme.Dark
-    if (stored == "light") return Theme.Light
-    // Fallback to system preference
-    val prefersDark = try {
-        window.matchMedia("(prefers-color-scheme: dark)").matches
-    } catch (_: Throwable) { false }
-    val t = if (prefersDark) Theme.Dark else Theme.Light
-    // Apply immediately so first render uses correct theme
-    applyTheme(t)
-    return t
-}
-
-private fun saveThemePreference(theme: Theme) {
-    try { window.localStorage.setItem("theme", if (theme == Theme.Dark) "dark" else "light") } catch (_: Throwable) {}
-}
-
-private fun applyTheme(theme: Theme) {
     // Toggle Bootstrap theme attribute
-    document.body?.setAttribute("data-bs-theme", if (theme == Theme.Dark) "dark" else "light")
+    document.body?.setAttribute("data-bs-theme", if (isDark) "dark" else "light")
     // Toggle GitHub Markdown CSS light/dark
     val light = document.getElementById("md-light") as? HTMLLinkElement
     val dark = document.getElementById("md-dark") as? HTMLLinkElement
-    if (theme == Theme.Dark) {
+    if (isDark) {
         light?.setAttribute("disabled", "")
         dark?.removeAttribute("disabled")
     } else {
@@ -421,6 +383,31 @@ private fun applyTheme(theme: Theme) {
     }
 }
 
+private fun initAutoTheme() {
+    val mql = try { window.matchMedia("(prefers-color-scheme: dark)") } catch (_: Throwable) { null }
+    if (mql == null) {
+        applyTheme(false)
+        return
+    }
+    // Set initial
+    applyTheme(mql.matches)
+    // React to changes (modern browsers)
+    try {
+        mql.addEventListener("change", { ev ->
+            val isDark = try { (ev.asDynamic().matches as Boolean) } catch (_: Throwable) { mql.matches }
+            applyTheme(isDark)
+        })
+    } catch (_: Throwable) {
+        // Legacy API fallback
+        try {
+            (mql.asDynamic()).addListener { mq: dynamic ->
+                val isDark = try { mq.matches as Boolean } catch (_: Throwable) { false }
+                applyTheme(isDark)
+            }
+        } catch (_: Throwable) {}
+    }
+}
+
 // Convert pseudo Markdown definition lists rendered by marked as paragraphs into proper <dl><dt><dd> structures.
 // Pattern supported (common in many MD flavors):
 // Term\n
@@ -527,6 +514,178 @@ private fun ensureBootstrapCodeBlocks(html: String): String {
     }
 }
 
+// ---- Lyng syntax highlighting over rendered HTML ----
+// This post-processor finds <pre><code class="language-lyng">…</code></pre> blocks and replaces the
+// inner code HTML with token-wrapped spans using the common Lyng highlighter.
+// It performs a minimal HTML entity decode on the code content to obtain the original text,
+// runs the highlighter, then escapes segments back and wraps with <span class="hl-…">.
+internal fun highlightLyngHtml(html: String): String {
+    // Regex to find <pre> ... <code class="language-lyng ...">(content)</code> ... </pre>
+    val preCodeRegex = Regex(
+        pattern = """<pre(\s+[^>]*)?>\s*<code([^>]*)>([\s\S]*?)</code>\s*</pre>""",
+        options = setOf(RegexOption.IGNORE_CASE)
+    )
+    val classAttrRegex = Regex("""\bclass\s*=\s*(["'])(.*?)\1""", RegexOption.IGNORE_CASE)
+
+    return preCodeRegex.replace(html) { m ->
+        val preAttrs = m.groups[1]?.value ?: ""
+        val codeAttrs = m.groups[2]?.value ?: ""
+        val codeHtml = m.groups[3]?.value ?: ""
+
+        val codeHasLyng = run {
+            val cls = classAttrRegex.find(codeAttrs)?.groupValues?.getOrNull(2) ?: ""
+            cls.split("\\s+".toRegex()).any { it.equals("language-lyng", ignoreCase = true) }
+        }
+        // If not explicitly Lyng, check if the <code> has any language class; if none, treat as Lyng by default
+        val hasAnyLanguage = run {
+            val cls = classAttrRegex.find(codeAttrs)?.groupValues?.getOrNull(2) ?: ""
+            cls.split("\\s+".toRegex()).any { it.startsWith("language-", ignoreCase = true) }
+        }
+
+        val treatAsLyng = codeHasLyng || !hasAnyLanguage
+        if (!treatAsLyng) return@replace m.value // leave untouched for non-Lyng languages
+
+        val text = htmlUnescape(codeHtml)
+
+        // If block has no explicit language (unfenced/indented), support doctest tail (trailing lines starting with ">>>")
+        val (headText, tailTextOrNull) = if (!codeHasLyng && !hasAnyLanguage) splitDoctestTail(text) else text to null
+
+        val headHighlighted = try {
+            applyLyngHighlightToText(headText)
+        } catch (_: Throwable) {
+            return@replace m.value
+        }
+        val tailHighlighted = tailTextOrNull?.let { renderDoctestTailAsComments(it) } ?: ""
+
+        val highlighted = headHighlighted + tailHighlighted
+
+        // Preserve original attrs; ensure <pre> has existing attrs (Bootstrap '.code' was handled earlier)
+        "<pre$preAttrs><code$codeAttrs>$highlighted</code></pre>"
+    }
+}
+
+// Split trailing doctest tail: consecutive lines at the end whose trimmedStart starts with ">>>".
+// Returns Pair(head, tail) where tail is null if no doctest lines found.
+private fun splitDoctestTail(text: String): Pair<String, String?> {
+    if (text.isEmpty()) return "" to null
+    // Normalize to \n for splitting; remember if original ended with newline
+    val hasTrailingNewline = text.endsWith("\n")
+    val lines = text.split("\n")
+    var i = lines.size - 1
+    // Skip trailing completely empty lines before looking for doctest markers
+    while (i >= 0 && lines[i].isEmpty()) i--
+    var count = 0
+    while (i >= 0) {
+        val line = lines[i]
+        // If last line is empty due to trailing newline, include it into tail only if there are already doctest lines
+        val trimmed = line.trimStart()
+        if (trimmed.isNotEmpty() && !trimmed.startsWith(">>>")) break
+        // Accept empty line only if it follows some doctest lines (keeps spacing), else stop
+        if (trimmed.isEmpty()) {
+            if (count == 0) break else { count++; i--; continue }
+        }
+        // doctest line
+        count++
+        i--
+    }
+    if (count == 0) return text to null
+    val splitIndex = lines.size - count
+    val head = buildString {
+        for (idx in 0 until splitIndex) {
+            append(lines[idx])
+            if (idx < lines.size - 1 || hasTrailingNewline) append('\n')
+        }
+    }
+    val tail = buildString {
+        for (idx in splitIndex until lines.size) {
+            append(lines[idx])
+            if (idx < lines.size - 1 || hasTrailingNewline) append('\n')
+        }
+    }
+    return head to tail
+}
+
+// Render the doctest tail as comment-highlighted lines. Expects the original textual tail including newlines.
+private fun renderDoctestTailAsComments(tail: String): String {
+    if (tail.isEmpty()) return ""
+    val sb = StringBuilder(tail.length + 32)
+    var start = 0
+    while (start <= tail.lastIndex) {
+        val nl = tail.indexOf('\n', start)
+        val line = if (nl >= 0) tail.substring(start, nl) else tail.substring(start)
+        // Wrap the whole line in comment styling
+        sb.append("<span class=\"hl-cmt\">")
+        sb.append(htmlEscape(line))
+        sb.append("</span>")
+        if (nl >= 0) sb.append('\n')
+        if (nl < 0) break else start = nl + 1
+    }
+    return sb.toString()
+}
+
+// Apply Lyng highlighter to raw code text, producing HTML with span classes.
+internal fun applyLyngHighlightToText(text: String): String {
+    val highlighter = net.sergeych.lyng.highlight.SimpleLyngHighlighter()
+    // Use spans as produced by the fixed lynglib highlighter (comments already extend to EOL there)
+    val spans = highlighter.highlight(text)
+    if (spans.isEmpty()) return htmlEscape(text)
+    val sb = StringBuilder(text.length + spans.size * 16)
+    var pos = 0
+    for (s in spans) {
+        if (s.range.start > pos) {
+            sb.append(htmlEscape(text.substring(pos, s.range.start)))
+        }
+        val cls = cssClassForKind(s.kind)
+        sb.append('<').append("span class=\"").append(cls).append('\"').append('>')
+        sb.append(htmlEscape(text.substring(s.range.start, s.range.endExclusive)))
+        sb.append("</span>")
+        pos = s.range.endExclusive
+    }
+    if (pos < text.length) sb.append(htmlEscape(text.substring(pos)))
+    return sb.toString()
+}
+
+// Note: No site-side span post-processing — we rely on lynglib's SimpleLyngHighlighter for correctness.
+
+private fun cssClassForKind(kind: net.sergeych.lyng.highlight.HighlightKind): String = when (kind) {
+    net.sergeych.lyng.highlight.HighlightKind.Keyword -> "hl-kw"
+    net.sergeych.lyng.highlight.HighlightKind.TypeName -> "hl-ty"
+    net.sergeych.lyng.highlight.HighlightKind.Identifier -> "hl-id"
+    net.sergeych.lyng.highlight.HighlightKind.Number -> "hl-num"
+    net.sergeych.lyng.highlight.HighlightKind.String -> "hl-str"
+    net.sergeych.lyng.highlight.HighlightKind.Char -> "hl-ch"
+    net.sergeych.lyng.highlight.HighlightKind.Regex -> "hl-rx"
+    net.sergeych.lyng.highlight.HighlightKind.Comment -> "hl-cmt"
+    net.sergeych.lyng.highlight.HighlightKind.Operator -> "hl-op"
+    net.sergeych.lyng.highlight.HighlightKind.Punctuation -> "hl-punc"
+    net.sergeych.lyng.highlight.HighlightKind.Label -> "hl-lbl"
+    net.sergeych.lyng.highlight.HighlightKind.Directive -> "hl-dir"
+    net.sergeych.lyng.highlight.HighlightKind.Error -> "hl-err"
+}
+
+// Minimal HTML escaping for text nodes
+private fun htmlEscape(s: String): String = buildString(s.length) {
+    for (ch in s) when (ch) {
+        '<' -> append("&lt;")
+        '>' -> append("&gt;")
+        '&' -> append("&amp;")
+        '"' -> append("&quot;")
+        '\'' -> append("&#39;")
+        else -> append(ch)
+    }
+}
+
+// Minimal unescape for code inner HTML produced by marked
+private fun htmlUnescape(s: String): String {
+    // handle common entities only
+    return s
+        .replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&amp;", "&")
+        .replace("&quot;", "\"")
+        .replace("&#39;", "'")
+}
+
 private fun rewriteImages(root: HTMLElement, basePath: String) {
     val imgs = root.querySelectorAll("img")
     for (i in 0 until imgs.length) {
@@ -624,6 +783,8 @@ fun activeIndexForTops(tops: List<Double>, offsetPx: Double): Int {
 }
 
 fun main() {
+    // Initialize automatic system theme before rendering UI
+    initAutoTheme()
     renderComposable(rootElementId = "root") { App() }
 }
 

site/src/jsMain/resources/index.html

@@ -46,15 +46,59 @@
       href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.min.css"
     />
     <style>
-      /* Visual polish for markdown area */
+      /* Unify markdown and page backgrounds with Bootstrap theme variables */
       .markdown-body {
         box-sizing: border-box;
         min-width: 200px;
         line-height: 1.6;
+        background: var(--bs-body-bg) !important;
+        color: var(--bs-body-color) !important;
       }
       .markdown-body > :first-child { margin-top: 0 !important; }
       .markdown-body table { margin: 1rem 0; }
-      .markdown-body pre { padding: .75rem; border-radius: .375rem; }
+      .markdown-body pre {
+        padding: .75rem;
+        border-radius: .375rem;
+        background: var(--bs-tertiary-bg);
+        border: 1px solid var(--bs-border-color);
+      }
+      /* Style only inline code, avoid affecting code blocks inside <pre> to prevent first-line extra indent */
+      .markdown-body :not(pre) > code:not([class]) {
+        background: var(--bs-tertiary-bg);
+        padding: .15rem .3rem;
+        border-radius: .25rem;
+      }
+
+      /* Lyng syntax highlighting palette (inspired by default GitHub/VS Code) */
+      /* Light theme */
+      .hl-kw  { color: #d73a49; font-weight: 600; }
+      .hl-ty  { color: #6f42c1; }
+      .hl-id  { color: #24292e; }
+      .hl-num { color: #005cc5; }
+      .hl-str { color: #032f62; }
+      .hl-ch  { color: #032f62; }
+      .hl-rx  { color: #116329; }
+      .hl-cmt { color: #6a737d; font-style: italic; }
+      .hl-op  { color: #8250df; }
+      .hl-punc{ color: #57606a; }
+      .hl-lbl { color: #e36209; }
+      .hl-dir { color: #6f42c1; }
+      .hl-err { color: #b31d28; text-decoration: underline wavy #b31d28; }
+
+      /* Dark theme overrides (GitHub Dark-like) */
+      [data-bs-theme="dark"] .hl-id   { color: #c9d1d9; }
+      [data-bs-theme="dark"] .hl-op   { color: #d2a8ff; }
+      [data-bs-theme="dark"] .hl-punc { color: #8b949e; }
+      [data-bs-theme="dark"] .hl-kw   { color: #ff7b72; }
+      [data-bs-theme="dark"] .hl-ty   { color: #d2a8ff; }
+      [data-bs-theme="dark"] .hl-num  { color: #79c0ff; }
+      [data-bs-theme="dark"] .hl-str,
+      [data-bs-theme="dark"] .hl-ch   { color: #a5d6ff; }
+      [data-bs-theme="dark"] .hl-rx   { color: #7ee787; }
+      [data-bs-theme="dark"] .hl-cmt  { color: #8b949e; }
+      [data-bs-theme="dark"] .hl-lbl  { color: #ffa657; }
+      [data-bs-theme="dark"] .hl-dir  { color: #d2a8ff; }
+      [data-bs-theme="dark"] .hl-err  { color: #ffa198; text-decoration-color: #ffa198; }
     </style>
   </head>
   <body>

site/src/jsTest/kotlin/CommentSpanExtendTest.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import kotlin.test.Test
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class CommentSpanExtendTest {
+    @Test
+    fun singleLineCommentCoversTillEol() {
+        val line = "// see the difference: apply changes this to newly created Point:"
+        val md = """
+            ```lyng
+            $line
+            ```
+        """.trimIndent()
+
+        val html = renderMarkdown(md)
+        // Entire line must be inside the comment span
+        assertTrue(html.contains("<span class=\"hl-cmt\">$line</span>"), "Comment should extend to EOL: $html")
+        // There must be no stray tail like </span>nt: (regression case)
+        assertFalse(html.contains("</span>nt:"), "No trailing tail should remain outside comment span: $html")
+    }
+}

site/src/jsTest/kotlin/HighlightSmokeTest.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.site
+
+import net.sergeych.lyng.highlight.HighlightKind
+import net.sergeych.lyng.highlight.HighlightSpan
+import net.sergeych.lyng.highlight.SimpleLyngHighlighter
+import kotlin.test.Test
+import kotlin.test.assertContains
+import kotlin.test.assertTrue
+
+class HighlightSmokeTest {
+
+    private fun spansToLabeled(text: String, spans: List<HighlightSpan>): List<Pair<String, HighlightKind>> =
+        spans.map { text.substring(it.range.start, it.range.endExclusive) to it.kind }
+
+    @Test
+    fun highlightAssertEqualsSnippet() {
+        val text = "assertEquals( [9,10], r.takeLast(2).toList() )"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+
+        // Basic sanity: identifier assertEquals present and not split
+        assertTrue(labeled.any { it.first == "assertEquals" && it.second == HighlightKind.Identifier })
+        // Basic numbers detection
+        assertTrue(labeled.any { it.first == "9" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "10" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "2" && it.second == HighlightKind.Number })
+    }
+
+    @Test
+    fun renderHtmlContainsCorrectClasses() {
+        val text = "assertEquals( [9,10], r.takeLast(2).toList() )"
+        val html = SiteHighlight.renderHtml(text)
+        // Ensure important parts are wrapped with expected classes
+        assertContains(html, "<span class=\"hl-id\">assertEquals</span>")
+        assertContains(html, "<span class=\"hl-num\">9</span>")
+        assertContains(html, "<span class=\"hl-num\">10</span>")
+        assertContains(html, "<span class=\"hl-num\">2</span>")
+        // Punctuation and operators appear; allow either combined or separate, just ensure class exists
+        assertTrue(html.contains("hl-punc"))
+        assertTrue(html.contains("hl-op"))
+    }
+
+}

site/src/jsTest/kotlin/LyngHighlightTest.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import kotlin.test.Test
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class LyngHighlightTest {
+    @Test
+    fun highlightsLyngFencedBlock() {
+        val md = """
+            ```lyng
+            and or not constructor property a+b
+            ```
+        """.trimIndent()
+        val html = renderMarkdown(md)
+
+        // Should produce span classes for keywords and operator
+        assertTrue(html.contains("hl-kw"), "Expected keyword class in highlighted Lyng block: $html")
+        assertTrue(html.contains("hl-op"), "Expected operator class in highlighted Lyng block: $html")
+        // Ensure code is inside <pre><code ... language-lyng>
+        assertTrue(html.contains("language-lyng", ignoreCase = true), "Expected language-lyng class retained: $html")
+    }
+
+    @Test
+    fun nonLyngBlocksAreUntouched() {
+        val md = """
+            ```kotlin
+            println("Hi")
+            ```
+        """.trimIndent()
+        val html = renderMarkdown(md)
+        // Should not include our Lyng-specific classes
+        assertFalse(html.contains("hl-kw"), "Non-Lyng block should not be Lyng-highlighted: $html")
+        assertTrue(html.contains("<pre"), "Expected <pre> present: $html")
+    }
+
+    @Test
+    fun escapesAngleBracketsInsideSpans() {
+        val md = """
+            ```lyng
+            a<b
+            ```
+        """.trimIndent()
+        val html = renderMarkdown(md)
+        // the '<' should be escaped in HTML
+        assertTrue(html.contains("&lt;"), "Expected escaped < inside highlighted HTML: $html")
+    }
+}

site/src/jsTest/kotlin/UnfencedLyngTest.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import kotlin.test.Test
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class UnfencedLyngTest {
+    @Test
+    fun indentedCodeBlocksAreLyngByDefault() {
+        val md = """
+            Some text paragraph.
+
+                and or not a+b
+
+            Next paragraph.
+        """.trimIndent()
+
+        val html = renderMarkdown(md)
+        // Should contain a code block and Lyng highlight classes
+        assertTrue(html.contains("<pre", ignoreCase = true), "Expected <pre> in rendered HTML: $html")
+        assertTrue(html.contains("<code", ignoreCase = true), "Expected <code> in rendered HTML: $html")
+        assertTrue(html.contains("hl-kw"), "Expected keyword highlight in indented code block: $html")
+        assertTrue(html.contains("hl-op"), "Expected operator highlight in indented code block: $html")
+        // Should not introduce a non-Lyng language class; language-lyng may or may not be present, but other languages shouldn't
+        assertFalse(html.contains("language-kotlin", ignoreCase = true), "Should not mark indented block as another language: $html")
+    }
+
+    @Test
+    fun doctestTailLinesRenderedAsComments() {
+        val md = """
+            Intro paragraph.
+
+                a + b
+                >>> 3
+                >>> ok
+        """.trimIndent()
+
+        val html = renderMarkdown(md)
+        // Lyng highlight should appear for the code line (the '+')
+        assertTrue(html.contains("hl-op"), "Expected operator highlighting for head part: $html")
+        // The doctest tail lines should be wrapped as comments
+        assertTrue(html.contains("hl-cmt"), "Expected comment highlighting for doctest tail: $html")
+        // Ensure the markers are inside the comment span content
+        assertTrue(html.contains("<span class=\"hl-cmt\">&gt;&gt;&gt;"), "Doctest lines should start with >>> inside comment span: $html")
+    }
+}