fixed highlighting and styles

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/CommentEolTest.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class CommentEolTest {
+
+    @Test
+    fun singleLineCommentExtendsToEol() {
+        val line = "// see the difference: apply changes this to newly created Point:"
+        val text = "$line\nnext"
+
+        val spans = SimpleLyngHighlighter().highlight(text)
+        // Find the comment span
+        val cmt = spans.firstOrNull { it.kind == HighlightKind.Comment }
+        assertTrue(cmt != null, "Expected a comment span")
+        // It should start at 0 and extend exactly to the end of the line (before \n)
+        val eol = text.indexOf('\n')
+        assertEquals(0, cmt!!.range.start, "Comment should start at column 0")
+        assertEquals(eol, cmt.range.endExclusive, "Comment should extend to EOL")
+        // Ensure there is no other span overlapping within the same line
+        spans.filter { it !== cmt }.forEach {
+            assertTrue(it.range.start >= eol, "No span should start before EOL for single-line comment")
+        }
+    }
+
+    @Test
+    fun blockCommentNotExtendedPastClosing() {
+        val text = "/* block */ rest"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val cmt = spans.firstOrNull { it.kind == HighlightKind.Comment }
+        assertTrue(cmt != null, "Expected a block comment span")
+        // The comment should end right after "/* block */"
+        val expectedEnd = "/* block */".length
+        assertEquals(expectedEnd, cmt!!.range.endExclusive, "Block comment should not be extended to EOL")
+    }
+
+    @Test
+    fun twoSingleLineCommentsEachToTheirEol() {
+        val text = "// first\n// second\nend"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val cmts = spans.filter { it.kind == HighlightKind.Comment }
+        assertEquals(2, cmts.size, "Expected two single-line comment spans")
+
+        val eol1 = text.indexOf('\n')
+        assertEquals(0, cmts[0].range.start)
+        assertEquals(eol1, cmts[0].range.endExclusive)
+
+        val line2Start = eol1 + 1
+        val eol2 = text.indexOf('\n', line2Start)
+        assertEquals(line2Start, cmts[1].range.start)
+        assertEquals(eol2, cmts[1].range.endExclusive)
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/HighlightMappingTest.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class HighlightMappingTest {
+
+    private fun spansToLabeled(text: String, spans: List<HighlightSpan>): List<Pair<String, HighlightKind>> =
+        spans.map { text.substring(it.range.start, it.range.endExclusive) to it.kind }
+
+    @Test
+    fun keywordsAndIdentifiers() {
+        val text = "a and b or not c"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+
+        // Expect identifiers and keywords in order
+        assertTrue(labeled.any { it.first == "a" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "and" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "b" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "or" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "not" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "c" && it.second == HighlightKind.Identifier })
+    }
+
+    @Test
+    fun reservedWordsConstructorPropertyAsKeywords() {
+        val text = "constructor property foo"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "constructor" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "property" && it.second == HighlightKind.Keyword })
+        assertTrue(labeled.any { it.first == "foo" && it.second == HighlightKind.Identifier })
+    }
+
+    @Test
+    fun numbersAndStringsAndChar() {
+        val text = "123 0xFF 1.0 'c' \"s\""
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "123" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first.lowercase() == "0xff" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "1.0" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "'c'" && it.second == HighlightKind.Char })
+        assertTrue(labeled.any { it.first == "\"s\"" && it.second == HighlightKind.String })
+    }
+
+    @Test
+    fun commentsHighlighted() {
+        val text = "// line\n/* block */"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first.startsWith("//") && it.second == HighlightKind.Comment })
+        assertTrue(labeled.any { it.first.startsWith("/*") && it.second == HighlightKind.Comment })
+    }
+
+    @Test
+    fun operatorsAndPunctuation() {
+        val text = "a+b; (x)"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        assertTrue(labeled.any { it.first == "+" && it.second == HighlightKind.Operator })
+        assertTrue(labeled.any { it.first == ";" && it.second == HighlightKind.Punctuation })
+        assertTrue(labeled.any { it.first == "(" && it.second == HighlightKind.Punctuation })
+        assertTrue(labeled.any { it.first == ")" && it.second == HighlightKind.Punctuation })
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/RegressionAssertEqualsTest.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Regression tests for highlighting real-world snippet mentioned in the issue.
+ */
+package net.sergeych.lyng.highlight
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class RegressionAssertEqualsTest {
+
+    private fun spansToLabeled(text: String, spans: List<HighlightSpan>): List<Pair<String, HighlightKind>> =
+        spans.map { text.substring(it.range.start, it.range.endExclusive) to it.kind }
+
+    @Test
+    fun assertEqualsSnippetIsTokenizedAndHighlightedSane() {
+        val text = "assertEquals( [9,10], r.takeLast(2).toList() )"
+        val spans = SimpleLyngHighlighter().highlight(text)
+        val labeled = spansToLabeled(text, spans)
+        // Debug print to help diagnose failures across targets
+        println("[DEBUG_LOG] labeled spans: " + labeled.joinToString(" | ") { "${it.first}:{${it.second}}" })
+
+        // Ensure identifier is not split: whole 'assertEquals' must be Identifier
+        assertTrue(labeled.any { it.first == "assertEquals" && it.second == HighlightKind.Identifier })
+
+        // Brackets and parentheses must be punctuation; spans may merge adjacent punctuation,
+        // so accept combined tokens like "()" or "],"
+        fun hasPunct(containing: Char) = labeled.any { containing in it.first && it.second == HighlightKind.Punctuation }
+        assertTrue(hasPunct('('))
+        assertTrue(hasPunct(')'))
+        assertTrue(hasPunct('['))
+        assertTrue(hasPunct(']'))
+        assertTrue(hasPunct(','))
+
+        // Numbers 9, 10 and 2 should be recognized as numbers
+        assertTrue(labeled.any { it.first == "9" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "10" && it.second == HighlightKind.Number })
+        assertTrue(labeled.any { it.first == "2" && it.second == HighlightKind.Number })
+
+        // Method chain identifiers and dots/operators
+        assertTrue(labeled.any { it.first == "." && it.second == HighlightKind.Operator })
+        assertTrue(labeled.any { it.first == "r" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "takeLast" && it.second == HighlightKind.Identifier })
+        assertTrue(labeled.any { it.first == "toList" && it.second == HighlightKind.Identifier })
+    }
+}

lynglib/src/commonTest/kotlin/net/sergeych/lyng/highlight/SourceOffsetTest.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Verify that Source/Pos → absolute offset mapping is consistent and preserves
+ * exact characters, including trailing spaces and Windows CRLF endings.
+ */
+package net.sergeych.lyng.highlight
+
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Source
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class SourceOffsetTest {
+
+    @Test
+    fun preservesTrailingSpacesInColumns() {
+        val txt = "abc  \nxyz" // two trailing spaces before the newline
+        val src = Source("snippet", txt)
+        // line 0: "abc  " length 5
+        val p = Pos(src, 0, 4) // column at the last space
+        val off = src.offsetOf(p)
+        assertEquals(4, off)
+        // Take substring from start to this pos, it must include two spaces
+        assertEquals("abc ", txt.substring(0, off))
+    }
+
+    @Test
+    fun crlfLineEndingsDoNotBreakOffsets() {
+        val txt = "a\r\nb\r\nc" // three lines, split by \n; \r remain at line ends
+        val src = Source("snippet", txt)
+        // Position at start of line 2 ('c')
+        val p = Pos(src, 2, 0)
+        val off = src.offsetOf(p)
+        // Offsets: line0 len=2 ("a\r"), plus one for \n, line1 len=2 ("b\r"), plus one for \n => 2+1+2+1=6
+        assertEquals(6, off)
+        assertEquals('c', txt[off])
+    }
+}