another attempt to fix plugin spellchecker

LYNG_AI_SPEC.md

@@ -1,9 +1,17 @@
-# Lyng Language AI Specification (V1.2)
+# Lyng Language AI Specification (V1.3)
 
 High-density specification for LLMs. Reference this for all Lyng code generation.
 
 ## 1. Core Philosophy & Syntax
-- **Everything is an Expression**: Blocks, `if`, `when`, `for`, `while` return their last expression (or `void`).
+- **Everything is an Expression**: Blocks, `if`, `when`, `for`, `while`, `do-while` return their last expression (or `void`).
+- **Loops with `else`**: `for`, `while`, and `do-while` support an optional `else` block.
+    - `else` executes **only if** the loop finishes normally (without a `break`).
+    - `break <value>` exits the loop and sets its return value.
+    - Loop Return Value:
+        1. Value from `break <value>`.
+        2. Result of `else` block (if loop finished normally and `else` exists).
+        3. Result of the last iteration (if loop finished normally and no `else`).
+        4. `void` (if loop body never executed and no `else`).
 - **Implicit Coroutines**: All functions are coroutines. No `async/await`. Use `launch { ... }` (returns `Deferred`) or `flow { ... }`.
 - **Variables**: `val` (read-only), `var` (mutable). Supports late-init `val` in classes (must be assigned in `init` or body).
 - **Null Safety**: `?` (nullable type), `?.` (safe access), `?( )` (safe invoke), `?{ }` (safe block invoke), `?[ ]` (safe index), `?:` or `??` (elvis), `?=` (assign-if-null).
@@ -87,3 +95,11 @@ val [first, middle..., last] = [1, 2, 3, 4, 5]
 // Safe Navigation and Elvis
 val companyName = person?.job?.company?.name ?? "Freelancer"
 ```
+
+## 8. Standard Library Discovery
+To collect data on the standard library and available APIs, AI should inspect:
+- **Global Symbols**: `lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt` (root functions like `println`, `sqrt`, `assert`).
+- **Core Type Members**: `lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/*.kt` (e.g., `ObjList.kt`, `ObjString.kt`, `ObjMap.kt`) for methods on built-in types.
+- **Lyng-side Extensions**: `lynglib/stdlib/lyng/root.lyng` for high-level functional APIs (e.g., `map`, `filter`, `any`, `lazy`).
+- **I/O & Processes**: `lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/` for `fs` and `process` modules.
+- **Documentation**: `docs/*.md` (e.g., `tutorial.md`, `lyngio.md`) for high-level usage and module overviews.

lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/annotators/LyngExternalAnnotator.kt

@@ -62,6 +62,7 @@ class LyngExternalAnnotator : ExternalAnnotator<LyngExternalAnnotator.Input, Lyn
         if (collectedInfo == null) return null
         ProgressManager.checkCanceled()
         val text = collectedInfo.text
+        val tokens = try { SimpleLyngHighlighter().highlight(text) } catch (_: Throwable) { emptyList() }
         
         // Use LyngAstManager to get the (potentially merged) Mini-AST
         val mini = LyngAstManager.getMiniAst(collectedInfo.file) 
@@ -224,8 +225,6 @@ class LyngExternalAnnotator : ExternalAnnotator<LyngExternalAnnotator.Input, Lyn
             // Heuristics on top of binder: function call-sites and simple name-based roles
             ProgressManager.checkCanceled()
 
-            val tokens = try { SimpleLyngHighlighter().highlight(text) } catch (_: Throwable) { emptyList() }
-
             // Build simple name -> role map for top-level vals/vars and parameters
             val nameRole = HashMap<String, com.intellij.openapi.editor.colors.TextAttributesKey>(8)
             mini.declarations.forEach { d ->
@@ -280,7 +279,6 @@ class LyngExternalAnnotator : ExternalAnnotator<LyngExternalAnnotator.Input, Lyn
 
         // Add annotation/label coloring using token highlighter
         run {
-            val tokens = try { SimpleLyngHighlighter().highlight(text) } catch (_: Throwable) { emptyList() }
             tokens.forEach { s ->
                 if (s.kind == HighlightKind.Label) {
                     val start = s.range.start
@@ -322,7 +320,6 @@ class LyngExternalAnnotator : ExternalAnnotator<LyngExternalAnnotator.Input, Lyn
 
         // Map Enum constants from token highlighter to IDEA enum constant color
         run {
-            val tokens = try { SimpleLyngHighlighter().highlight(text) } catch (_: Throwable) { emptyList() }
             tokens.forEach { s ->
                 if (s.kind == HighlightKind.EnumConstant) {
                     val start = s.range.start
@@ -334,24 +331,10 @@ class LyngExternalAnnotator : ExternalAnnotator<LyngExternalAnnotator.Input, Lyn
             }
         }
 
-        // Build spell index payload: identifiers from symbols + references; comments/strings from simple highlighter
+        // Build spell index payload: identifiers + comments/strings from simple highlighter.
-        val idRanges = mutableSetOf<IntRange>()
+        // We use the highlighter as the source of truth for all "words" to check, including
-        try {
+        // identifiers that might not be bound by the Binder.
-            val binding = Binder.bind(text, mini)
+        val idRanges = tokens.filter { it.kind == HighlightKind.Identifier }.map { it.range.start until it.range.endExclusive }
-            binding.symbols.forEach { sym ->
-                val s = sym.declStart
-                val e = sym.declEnd
-                if (s in 0..e && e <= text.length && s < e) idRanges += (s until e)
-            }
-            binding.references.forEach { ref ->
-                val s = ref.start
-                val e = ref.end
-                if (s in 0..e && e <= text.length && s < e) idRanges += (s until e)
-            }
-        } catch (_: Throwable) {
-            // Best-effort; no identifiers if binder fails
-        }
-        val tokens = try { SimpleLyngHighlighter().highlight(text) } catch (_: Throwable) { emptyList() }
         val commentRanges = tokens.filter { it.kind == HighlightKind.Comment }.map { it.range.start until it.range.endExclusive }
         val stringRanges = tokens.filter { it.kind == HighlightKind.String }.map { it.range.start until it.range.endExclusive }
 

lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/grazie/LyngGrazieAnnotator.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ * Copyright 2026 Sergey S. Chernov real.sergeych@gmail.com
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -150,16 +150,18 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
         }
         log.info("LyngGrazieAnnotator.apply: used=${chosenEntry ?: "<none>"}, totalFindings=$totalReturned, painting=${findings.size}")
 
-        // IMPORTANT: Do NOT fallback to the tiny bundled vocabulary on modern IDEs.
-        // If Grazie/Natural Languages processing returned nothing, we simply exit here
-        // to avoid low‑quality results from the legacy dictionary.
-        if (findings.isEmpty()) return
-
         for (f in findings) {
             val ab = holder.newAnnotation(HighlightSeverity.INFORMATION, f.message).range(f.range)
             applyTypoStyleIfRequested(file, ab)
             ab.create()
         }
+
+        // SUPPLEMENT: Always run the fallback spellchecker to ensure spelling errors are not ignored.
+        // It will avoid duplicating findings already reported by Grazie.
+        val painted = fallbackWithLegacySpellcheckerIfAvailable(file, fragments, holder, findings)
+        if (painted > 0) {
+            log.info("LyngGrazieAnnotator.apply: supplemented with $painted typos from legacy engine")
+        }
     }
 
     private fun scheduleOneShotRestart(file: PsiFile, modStamp: Long) {
@@ -381,7 +383,8 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
     private fun fallbackWithLegacySpellcheckerIfAvailable(
         file: PsiFile,
         fragments: List<Pair<TextContent, TextRange>>,
-        holder: AnnotationHolder
+        holder: AnnotationHolder,
+        existingFindings: List<Finding>
     ): Int {
         return try {
             val mgrCls = Class.forName("com.intellij.spellchecker.SpellCheckerManager")
@@ -389,12 +392,12 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
             val isCorrect = mgrCls.methods.firstOrNull { it.name == "isCorrect" && it.parameterCount == 1 && it.parameterTypes[0] == String::class.java }
             if (getInstance == null || isCorrect == null) {
                 // No legacy spellchecker API available — fall back to naive painter
-                return naiveFallbackPaint(file, fragments, holder)
+                return naiveFallbackPaint(file, fragments, holder, existingFindings)
             }
             val mgr = getInstance.invoke(null, file.project)
             if (mgr == null) {
                 // Legacy manager not present for this project — use naive fallback
-                return naiveFallbackPaint(file, fragments, holder)
+                return naiveFallbackPaint(file, fragments, holder, existingFindings)
             }
             var painted = 0
             val docText = file.viewProvider.document?.text ?: return 0
@@ -411,13 +414,18 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
                     for (part in parts) {
                         if (part.length <= 2) continue
                         if (isAllowedWord(part)) continue
-                        // Quick allowlist for very common words to reduce noise if dictionaries differ
+
-                        val ok = try { isCorrect.invoke(mgr, part) as? Boolean } catch (_: Throwable) { null }
-                        if (ok == false) {
                         // Map part back to original token occurrence within this hostRange
                         val localStart = m.range.first + token.indexOf(part)
                         val localEnd = localStart + part.length
                         val abs = TextRange(hostRange.startOffset + localStart, hostRange.startOffset + localEnd)
+
+                        // Avoid duplicating findings from Grazie
+                        if (existingFindings.any { it.range.intersects(abs) }) continue
+
+                        // Quick allowlist for very common words to reduce noise if dictionaries differ
+                        val ok = try { isCorrect.invoke(mgr, part) as? Boolean } catch (_: Throwable) { null }
+                        if (ok == false) {
                             paintTypoAnnotation(file, holder, abs, part)
                             painted++
                             flagged++
@@ -430,14 +438,15 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
             painted
         } catch (_: Throwable) {
             // If legacy manager is not available, fall back to a very naive heuristic (no external deps)
-            return naiveFallbackPaint(file, fragments, holder)
+            return naiveFallbackPaint(file, fragments, holder, existingFindings)
         }
     }
 
     private fun naiveFallbackPaint(
         file: PsiFile,
         fragments: List<Pair<TextContent, TextRange>>,
-        holder: AnnotationHolder
+        holder: AnnotationHolder,
+        existingFindings: List<Finding>
     ): Int {
         var painted = 0
         val docText = file.viewProvider.document?.text
@@ -461,6 +470,14 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
                     seen++
                     val lower = part.lowercase()
                     if (lower.length <= 2 || isAllowedWord(part)) continue
+
+                    val localStart = m.range.first + token.indexOf(part)
+                    val localEnd = localStart + part.length
+                    val abs = TextRange(hostRange.startOffset + localStart, hostRange.startOffset + localEnd)
+
+                    // Avoid duplicating findings from Grazie
+                    if (existingFindings.any { it.range.intersects(abs) }) continue
+
                     // Heuristic: no vowels OR 3 repeated chars OR ends with unlikely double consonants
                     val noVowel = lower.none { it in "aeiouy" }
                     val triple = Regex("(.)\\1\\1").containsMatchIn(lower)
@@ -480,9 +497,6 @@ class LyngGrazieAnnotator : ExternalAnnotator<LyngGrazieAnnotator.Input, LyngGra
                         }
                     }
                     if (looksWrong) {
-                        val localStart = m.range.first + token.indexOf(part)
-                        val localEnd = localStart + part.length
-                        val abs = TextRange(hostRange.startOffset + localStart, hostRange.startOffset + localEnd)
                         paintTypoAnnotation(file, holder, abs, part)
                         painted++
                         flagged++

lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/highlight/LyngLexer.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
+ * Copyright 2026 Sergey S. Chernov real.sergeych@gmail.com
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -101,8 +101,9 @@ class LyngLexer : LexerBase() {
             return
         }
 
-        // String "..." with simple escape handling
+        // String "..." or '...' with simple escape handling
-        if (ch == '"') {
+        if (ch == '"' || ch == '\'') {
+            val quote = ch
             i++
             while (i < endOffset) {
                 val c = buffer[i]
@@ -110,7 +111,7 @@ class LyngLexer : LexerBase() {
                     i += 2
                     continue
                 }
-                if (c == '"') { i++; break }
+                if (c == quote) { i++; break }
                 i++
             }
             myTokenEnd = i

lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/spell/LyngSpellcheckingStrategy.kt

@@ -17,9 +17,6 @@
 package net.sergeych.lyng.idea.spell
 
 // Avoid Tokenizers helper to keep compatibility; implement our own tokenizers
-import com.intellij.ide.plugins.PluginManagerCore
-import com.intellij.openapi.diagnostic.Logger
-import com.intellij.openapi.extensions.PluginId
 import com.intellij.openapi.util.TextRange
 import com.intellij.psi.PsiElement
 import com.intellij.spellchecker.inspections.PlainTextSplitter
@@ -38,67 +35,21 @@ import net.sergeych.lyng.idea.settings.LyngFormatterSettings
  */
 class LyngSpellcheckingStrategy : SpellcheckingStrategy() {
 
-    private val log = Logger.getInstance(LyngSpellcheckingStrategy::class.java)
-    @Volatile private var loggedOnce = false
-
-    private fun grazieInstalled(): Boolean {
-        // Support both historical and bundled IDs
-        return PluginManagerCore.isPluginInstalled(PluginId.getId("com.intellij.grazie")) ||
-                PluginManagerCore.isPluginInstalled(PluginId.getId("tanvd.grazi"))
-    }
-
-    private fun grazieApiAvailable(): Boolean = try {
-        // If this class is absent (as in IC-243), third-party plugins can't run Grazie programmatically
-        Class.forName("com.intellij.grazie.grammar.GrammarChecker")
-        true
-    } catch (_: Throwable) { false }
-
     override fun getTokenizer(element: PsiElement): Tokenizer<*> {
         if (element is com.intellij.psi.PsiFile) return EMPTY_TOKENIZER
 
-        val hasGrazie = grazieInstalled()
-        val hasGrazieApi = grazieApiAvailable()
         val settings = LyngFormatterSettings.getInstance(element.project)
-        if (!loggedOnce) {
-            loggedOnce = true
-            log.info("LyngSpellcheckingStrategy activated: hasGrazie=$hasGrazie, grazieApi=$hasGrazieApi, preferGrazieForCommentsAndLiterals=${settings.preferGrazieForCommentsAndLiterals}, spellCheckStringLiterals=${settings.spellCheckStringLiterals}, grazieChecksIdentifiers=${settings.grazieChecksIdentifiers}")
-        }
-
-        val file = element.containingFile ?: return EMPTY_TOKENIZER
         val et = element.node?.elementType
-        val index = LyngSpellIndex.getUpToDate(file)
 
-        // Decide responsibility per settings
+        if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.IDENTIFIER || et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.LABEL) {
-        // If Grazie is present but its public API is not available (IC-243), do NOT delegate to it.
+            return IDENTIFIER_TOKENIZER
-        val preferGrazie = hasGrazie && hasGrazieApi && settings.preferGrazieForCommentsAndLiterals
-        val grazieIds = hasGrazie && hasGrazieApi && settings.grazieChecksIdentifiers
-
-        if (index == null) {
-            // Index not ready: fall back to Lexer-based token types.
-            // Identifiers are safe because LyngLexer separates keywords from identifiers.
-            if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.IDENTIFIER) {
-                return if (grazieIds) EMPTY_TOKENIZER else IDENTIFIER_TOKENIZER
         }
         if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.LINE_COMMENT || et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.BLOCK_COMMENT) {
-                return if (preferGrazie) EMPTY_TOKENIZER else COMMENT_TEXT_TOKENIZER
+            return COMMENT_TEXT_TOKENIZER
         }
         if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.STRING && settings.spellCheckStringLiterals) {
-                return if (preferGrazie) EMPTY_TOKENIZER else STRING_WITH_PRINTF_EXCLUDES
+            return STRING_WITH_PRINTF_EXCLUDES
         }
-            return EMPTY_TOKENIZER
-        }
-
-        val elRange = element.textRange ?: return EMPTY_TOKENIZER
-        fun overlaps(list: List<TextRange>) = list.any { it.intersects(elRange) }
-
-        // Identifiers: only if range is within identifiers index and not delegated to Grazie
-        if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.IDENTIFIER && overlaps(index.identifiers) && !grazieIds) return IDENTIFIER_TOKENIZER
-
-        // Comments: only if not delegated to Grazie and overlapping indexed comments
-        if ((et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.LINE_COMMENT || et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.BLOCK_COMMENT) && overlaps(index.comments) && !preferGrazie) return COMMENT_TEXT_TOKENIZER
-
-        // Strings: only if not delegated to Grazie, literals checking enabled, and overlapping indexed strings
-        if (et == net.sergeych.lyng.idea.highlight.LyngTokenTypes.STRING && settings.spellCheckStringLiterals && overlaps(index.strings) && !preferGrazie) return STRING_WITH_PRINTF_EXCLUDES
 
         return EMPTY_TOKENIZER
     }

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -4796,5 +4796,20 @@ class ScriptTest {
             assertEquals("r:r", T().l("r"))
         """.trimIndent())
     }
+
+    @Test
+    fun testTypedArgsWithInitializers() = runTest {
+        eval("""
+            fun f(a: String = "foo") = a + "!"
+            fun g(a: String? = null) = a ?: "!!"
+            assertEquals(f(), "foo!")
+            assertEquals(f(), "!!")
+            assertEquals(f("bar"), "bar!")
+            class T(b: Int=42,c: String?=null) 
+            assertEquals(42, T().b)
+            assertEquals(null, T().c)
+        """.trimIndent())
+    }
+
 }