fixed highlighting and styles

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt

@@ -84,12 +84,12 @@ private class Parser(fromPos: Pos) {
                 when (currentChar) {
                     '+' -> {
                         pos.advance()
-                        Token("+", from, Token.Type.PLUS2)
+                        Token("++", from, Token.Type.PLUS2)
                     }
 
                     '=' -> {
                         pos.advance()
-                        Token("+", from, Token.Type.PLUSASSIGN)
+                        Token("+=", from, Token.Type.PLUSASSIGN)
                     }
 
                     else ->
@@ -106,7 +106,7 @@ private class Parser(fromPos: Pos) {
 
                     '=' -> {
                         pos.advance()
-                        Token("-", from, Token.Type.MINUSASSIGN)
+                        Token("-=", from, Token.Type.MINUSASSIGN)
                     }
 
                     '>' -> {
@@ -129,17 +129,17 @@ private class Parser(fromPos: Pos) {
             '/' -> when (currentChar) {
                 '/' -> {
                     pos.advance()
-                    Token(loadToEndOfLine().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
+                    val body = loadToEndOfLine()
+                    // Include the leading '//' and do not trim; keep exact lexeme (excluding preceding codepoint)
+                    Token("//" + body, from, Token.Type.SINLGE_LINE_COMMENT)
                 }
 
                 '*' -> {
                     pos.advance()
-                    Token(
-                        loadTo("*/")?.trim()
-                            ?: throw ScriptError(from, "Unterminated multiline comment"),
-                        from,
-                        Token.Type.MULTILINE_COMMENT
-                    )
+                    val content = loadTo("*/")
+                        ?: throw ScriptError(from, "Unterminated multiline comment")
+                    // loadTo consumes the closing fragment, so we are already after */
+                    Token("/*" + content + "*/", from, Token.Type.MULTILINE_COMMENT)
                 }
 
                 '=' -> {
@@ -403,7 +403,8 @@ private class Parser(fromPos: Pos) {
             // could be integer, also hex:
             if (currentChar == 'x' && p1 == "0") {
                 pos.advance()
-                Token(loadChars { it in hexDigits }, start, Token.Type.HEX).also {
+                val hex = loadChars { it in hexDigits }
+                Token(hex, start, Token.Type.HEX).also {
                     if (currentChar.isLetter())
                         raise("invalid hex literal")
                 }
@@ -541,11 +542,11 @@ private class Parser(fromPos: Pos) {
 
     private fun loadToEndOfLine(): String {
         val result = StringBuilder()
-        val l = pos.line
-        do {
+        // Read characters up to but not including the line break
+        while (!pos.end && pos.currentChar != '\n') {
             result.append(pos.currentChar)
             pos.advance()
-        } while (pos.line == l)
+        }
         return result.toString()
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pos.kt

@@ -97,8 +97,5 @@ class MutablePos(private val from: Pos) {
     }
 
     override fun toString() = "($line:$column)"
-
-    init {
-        if( lines[0].isEmpty()) advance()
-    }
+    
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Source.kt

@@ -19,9 +19,13 @@ package net.sergeych.lyng
 
 import net.sergeych.lyng.obj.ObjString
 
-class Source(val fileName: String, text: String) {
+class Source(val fileName: String, val text: String) {
 
-    val lines = text.lines().map { it.trimEnd() }
+    // Preserve original text characters exactly; do not trim trailing spaces.
+    // Split by "\n" boundaries as the lexer models line breaks uniformly as a single newline
+    // between logical lines, regardless of original platform line endings.
+    // We intentionally do NOT trim line ends to keep columns accurate.
+    val lines: List<String> = text.split('\n')
 
     val objSourceName by lazy { ObjString(fileName) }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/highlight/HighlightApi.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Cross-platform highlighting API for the Lyng language.
+ */
+package net.sergeych.lyng.highlight
+
+/** Represents a half-open character range [start, endExclusive). */
+data class TextRange(val start: Int, val endExclusive: Int) {
+    init { require(start <= endExclusive) { "Invalid range: $start..$endExclusive" } }
+}
+
+/** Kinds of tokens for syntax highlighting. */
+enum class HighlightKind {
+    Keyword,
+    TypeName,
+    Identifier,
+    Number,
+    String,
+    Char,
+    Regex,
+    Comment,
+    Operator,
+    Punctuation,
+    Label,
+    Directive,
+    Error,
+}
+
+/** A highlighted span: character range and its semantic/lexical kind. */
+data class HighlightSpan(val range: TextRange, val kind: HighlightKind)
+
+/** Base interface for Lyng syntax highlighters. */
+interface LyngHighlighter {
+    /**
+     * Produce highlight spans for the given [text]. Spans are non-overlapping and
+     * ordered by start position. Adjacent spans of the same kind may be merged.
+     */
+    fun highlight(text: String): List<HighlightSpan>
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/highlight/SimpleLyngHighlighter.kt

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package net.sergeych.lyng.highlight
+
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.Token.Type
+import net.sergeych.lyng.parseLyng
+
+/** Extension that converts a [Pos] (line/column) into absolute character offset in the [Source] text. */
+fun Source.offsetOf(pos: Pos): Int {
+    var off = 0
+    // Sum full preceding lines + one '\n' per line (lines[] were created by String.lines())
+    var i = 0
+    while (i < pos.line) {
+        off += lines[i].length + 1 // assume \n as separator
+        i++
+    }
+    off += pos.column
+    return off
+}
+
+private val reservedIdKeywords = setOf("constructor", "property")
+// Fallback textual keywords that might come from the lexer as ID in some contexts (e.g., snippets)
+private val fallbackKeywordIds = setOf("and", "or", "not")
+
+/** Maps lexer token type (and sometimes value) to a [HighlightKind]. */
+private fun kindOf(type: Type, value: String): HighlightKind? = when (type) {
+    // identifiers and reserved ids
+    Type.ID -> when {
+        value in reservedIdKeywords -> HighlightKind.Keyword
+        value.lowercase() in fallbackKeywordIds -> HighlightKind.Keyword
+        else -> HighlightKind.Identifier
+    }
+
+    // numbers
+    Type.INT, Type.REAL, Type.HEX -> HighlightKind.Number
+
+    // text literals
+    Type.STRING, Type.STRING2 -> HighlightKind.String
+    Type.CHAR -> HighlightKind.Char
+    Type.REGEX -> HighlightKind.Regex
+
+    // comments
+    Type.SINLGE_LINE_COMMENT, Type.MULTILINE_COMMENT -> HighlightKind.Comment
+
+    // punctuation
+    Type.LPAREN, Type.RPAREN, Type.LBRACE, Type.RBRACE, Type.LBRACKET, Type.RBRACKET,
+    Type.COMMA, Type.SEMICOLON, Type.COLON -> HighlightKind.Punctuation
+
+    // textual control keywords
+    Type.IN, Type.NOTIN, Type.IS, Type.NOTIS, Type.AS, Type.ASNULL,
+    Type.AND, Type.OR, Type.NOT -> HighlightKind.Keyword
+
+    // labels / annotations
+    Type.LABEL, Type.ATLABEL -> HighlightKind.Label
+
+    // operators and symbolic constructs
+    Type.PLUS, Type.MINUS, Type.STAR, Type.SLASH, Type.PERCENT,
+    Type.ASSIGN, Type.PLUSASSIGN, Type.MINUSASSIGN, Type.STARASSIGN, Type.SLASHASSIGN, Type.PERCENTASSIGN,
+    Type.PLUS2, Type.MINUS2,
+    Type.EQ, Type.NEQ, Type.LT, Type.LTE, Type.GT, Type.GTE, Type.REF_EQ, Type.REF_NEQ, Type.MATCH, Type.NOTMATCH,
+    Type.DOT, Type.ARROW, Type.EQARROW, Type.QUESTION, Type.COLONCOLON,
+    Type.SHL, Type.SHR, Type.ELLIPSIS, Type.DOTDOT, Type.DOTDOTLT,
+    Type.NULL_COALESCE, Type.ELVIS, Type.NULL_COALESCE_INDEX, Type.NULL_COALESCE_INVOKE, Type.NULL_COALESCE_BLOCKINVOKE,
+    Type.SHUTTLE,
+    // bitwise textual operators (treat as operators for visuals)
+    Type.BITAND, Type.BITOR, Type.BITXOR, Type.BITNOT -> HighlightKind.Operator
+
+    // non-highlighting tokens
+    Type.NEWLINE, Type.EOF -> null
+}
+
+/** Merge contiguous spans of the same [HighlightKind] to reduce output size. */
+private fun mergeAdjacent(spans: List<HighlightSpan>): List<HighlightSpan> {
+    if (spans.isEmpty()) return spans
+    val out = ArrayList<HighlightSpan>(spans.size)
+    var prev = spans[0]
+    for (i in 1 until spans.size) {
+        val cur = spans[i]
+        if (cur.kind == prev.kind && cur.range.start == prev.range.endExclusive) {
+            prev = HighlightSpan(TextRange(prev.range.start, cur.range.endExclusive), prev.kind)
+        } else {
+            out += prev
+            prev = cur
+        }
+    }
+    out += prev
+    return out
+}
+
+/** Simple highlighter using the existing Lyng lexer (no incremental support yet). */
+class SimpleLyngHighlighter : LyngHighlighter {
+    override fun highlight(text: String): List<HighlightSpan> {
+        val src = Source("<snippet>", text)
+        val tokens = parseLyng(src)
+        val raw = ArrayList<HighlightSpan>(tokens.size)
+        fun adjustQuoteSpan(startOffset: Int, quoteChar: Char): TextRange {
+            var s = startOffset
+            if (s > 0 && text[s - 1] == quoteChar) s -= 1
+            var i = s + 1
+            while (i < text.length) {
+                val ch = text[i]
+                if (ch == '\\') {
+                    i += if (i + 1 < text.length) 2 else 1
+                    continue
+                }
+                if (ch == quoteChar) {
+                    return TextRange(s, i + 1)
+                }
+                i++
+            }
+            // Unterminated, highlight till end
+            return TextRange(s, text.length)
+        }
+
+        for (t in tokens) {
+            val k = kindOf(t.type, t.value) ?: continue
+            val start = src.offsetOf(t.pos)
+            val range = when (t.type) {
+                Type.STRING, Type.STRING2 -> adjustQuoteSpan(start, '"')
+                Type.CHAR -> adjustQuoteSpan(start, '\'')
+                else -> TextRange(start, (start + t.value.length).coerceAtMost(text.length))
+            }
+            if (range.endExclusive > range.start) raw += HighlightSpan(range, k)
+        }
+        // Adjust single-line comment spans to extend till EOL to compensate for lexer offset/length quirks
+        val adjusted = extendSingleLineCommentsToEol(text, raw)
+        // Spans are in order; merge adjacent of the same kind for compactness
+        return mergeAdjacent(adjusted)
+    }
+}
+
+/**
+ * Workaround/fix: ensure that single-line comment spans that start with `//` extend until the end of line.
+ * Drops any subsequent spans that would overlap the extended comment on the same line.
+ */
+private fun extendSingleLineCommentsToEol(
+    text: String,
+    spans: List<HighlightSpan>
+): List<HighlightSpan> {
+    if (spans.isEmpty()) return spans
+    val out = ArrayList<HighlightSpan>(spans.size)
+    var i = 0
+    while (i < spans.size) {
+        val s = spans[i]
+        if (s.kind == HighlightKind.Comment) {
+            // Check the original text actually has '//' at span start to avoid touching block comments
+            val start = s.range.start
+            val ahead = if (start in text.indices) text.substring(start, minOf(text.length, start + 2)) else ""
+            if (ahead == "//") {
+                // Extend to end of current line
+                val eol = text.indexOf('\n', start)
+                val newEnd = if (eol >= 0) eol else text.length
+                var j = i + 1
+                while (j < spans.size && spans[j].range.start < newEnd) {
+                    // Consume all overlapping spans on this line
+                    j++
+                }
+                out += HighlightSpan(TextRange(start, newEnd), s.kind)
+                i = j
+                continue
+            }
+        }
+        out += s
+        i++
+    }
+    return out
+}