lynglib: added MiniAST support

lyngweb: better syntax highlighting and code editor
This commit is contained in:
sergeych committed 2025-11-24 00:38:17 +01:00
1 parent 28b961d339
commit ea0ecb1db3
24 files changed
+3095 -218

No files matched your search

@@ -17,6 +17,7 @@
package net.sergeych.lyng
import net.sergeych.lyng.miniast.MiniTypeRef
import net.sergeych.lyng.obj.Obj
import net.sergeych.lyng.obj.ObjList
import net.sergeych.lyng.obj.ObjRecord
@@ -135,6 +136,7 @@ data class ArgsDeclaration(val params: List<Item>, val endTokenType: Token.Type)
data class Item(
val name: String,
val type: TypeDecl = TypeDecl.TypeAny,
val miniType: MiniTypeRef? = null,
val pos: Pos = Pos.builtIn,
val isEllipsis: Boolean = false,
/**
@@ -18,6 +18,8 @@
package net.sergeych.lyng
import ObjEnumClass
import net.sergeych.lyng.Compiler.Companion.compile
import net.sergeych.lyng.miniast.*
import net.sergeych.lyng.obj.*
import net.sergeych.lyng.pacman.ImportProvider
@@ -61,7 +63,59 @@ class Compiler(
var packageName: String? = null
class Settings
class Settings(
val miniAstSink: MiniAstSink? = null,
)
// Optional sink for mini-AST streaming (null by default, zero overhead when not used)
private val miniSink: MiniAstSink? = settings.miniAstSink
// --- Doc-comment collection state (for immediate preceding declarations) ---
private val pendingDocLines = mutableListOf<String>()
private var pendingDocStart: Pos? = null
private var prevWasComment: Boolean = false
private fun stripCommentLexeme(raw: String): String {
return when {
raw.startsWith("//") -> raw.removePrefix("//")
raw.startsWith("/*") && raw.endsWith("*/") -> {
val inner = raw.substring(2, raw.length - 2)
// Trim leading "*" prefixes per line like Javadoc style
inner.lines().joinToString("\n") { line ->
val t = line.trimStart()
if (t.startsWith("*")) t.removePrefix("*").trimStart() else line
}
}
else -> raw
}
}
private fun pushPendingDocToken(t: Token) {
val s = stripCommentLexeme(t.value)
if (pendingDocStart == null) pendingDocStart = t.pos
pendingDocLines += s
prevWasComment = true
}
private fun clearPendingDoc() {
pendingDocLines.clear()
pendingDocStart = null
prevWasComment = false
}
private fun consumePendingDoc(): MiniDoc? {
if (pendingDocLines.isEmpty()) return null
val raw = pendingDocLines.joinToString("\n").trimEnd()
val summary = raw.lines().firstOrNull { it.isNotBlank() }?.trim()
val start = pendingDocStart ?: cc.currentPos()
val doc = MiniDoc(MiniRange(start, start), raw = raw, summary = summary)
clearPendingDoc()
return doc
}
// Set just before entering a declaration parse, taken from keyword token position
private var pendingDeclStart: Pos? = null
private var pendingDeclDoc: MiniDoc? = null
private val initStack = mutableListOf<MutableList<Statement>>()
@@ -75,6 +129,9 @@ class Compiler(
private val codeContexts = mutableListOf<CodeContext>(CodeContext.Module(null))
// Last parsed block range (for Mini-AST function body attachment)
private var lastParsedBlockRange: MiniRange? = null
private suspend fun <T> inCodeContext(context: CodeContext, f: suspend () -> T): T {
return try {
codeContexts.add(context)
@@ -90,9 +147,19 @@ class Compiler(
// Track locals at script level for fast local refs
return withLocalNames(emptySet()) {
// package level declarations
// Notify sink about script start
miniSink?.onScriptStart(start)
do {
val t = cc.current()
if (t.type == Token.Type.NEWLINE || t.type == Token.Type.SINLGE_LINE_COMMENT || t.type == Token.Type.MULTILINE_COMMENT) {
when (t.type) {
Token.Type.SINLGE_LINE_COMMENT, Token.Type.MULTILINE_COMMENT -> pushPendingDocToken(t)
Token.Type.NEWLINE -> {
// A standalone newline not immediately following a comment resets doc buffer
if (!prevWasComment) clearPendingDoc() else prevWasComment = false
}
else -> {}
}
cc.next()
continue
}
@@ -114,6 +181,30 @@ class Compiler(
cc.next()
val pos = cc.currentPos()
val name = loadQualifiedName()
// Emit MiniImport with approximate per-segment ranges
run {
try {
val parts = name.split('.')
if (parts.isNotEmpty()) {
var col = pos.column
val segs = parts.map { p ->
val start = Pos(pos.source, pos.line, col)
val end = Pos(pos.source, pos.line, col + p.length)
col += p.length + 1 // account for following '.' between segments
net.sergeych.lyng.miniast.MiniImport.Segment(p, net.sergeych.lyng.miniast.MiniRange(start, end))
}
val lastEnd = segs.last().range.end
miniSink?.onImport(
net.sergeych.lyng.miniast.MiniImport(
net.sergeych.lyng.miniast.MiniRange(pos, lastEnd),
segs
)
)
}
} catch (_: Throwable) {
// best-effort; ignore import mini emission failures
}
}
val module = importManager.prepareImport(pos, name, null)
statements += statement {
module.importInto(this, null)
@@ -122,6 +213,17 @@ class Compiler(
continue
}
}
// Fast-path: top-level function declarations. Handle here to ensure
// Mini-AST emission even if qualifier matcher paths change.
if (t.value == "fun" || t.value == "fn") {
// Consume the keyword and delegate to the function parser
cc.next()
pendingDeclStart = t.pos
pendingDeclDoc = consumePendingDoc()
val st = parseFunctionDeclaration(isOpen = false, isExtern = false, isStatic = false)
statements += st
continue
}
}
val s = parseStatement(braceMeansLambda = true)?.also {
statements += it
@@ -137,6 +239,9 @@ class Compiler(
} while (true)
Script(start, statements)
}.also {
// Best-effort script end notification (use current position)
miniSink?.onScriptEnd(cc.currentPos(), net.sergeych.lyng.miniast.MiniScript(MiniRange(start, cc.currentPos())))
}
}
@@ -608,12 +713,13 @@ class Compiler(
cc.ifNextIs(Token.Type.ASSIGN) {
defaultValue = parseExpression()
}
// type information
val typeInfo = parseTypeDeclaration()
// type information (semantic + mini syntax)
val (typeInfo, miniType) = parseTypeDeclarationWithMini()
val isEllipsis = cc.skipTokenOfType(Token.Type.ELLIPSIS, isOptional = true)
result += ArgsDeclaration.Item(
t.value,
typeInfo,
miniType,
t.pos,
isEllipsis,
defaultValue,
@@ -657,11 +763,87 @@ class Compiler(
}
private fun parseTypeDeclaration(): TypeDecl {
return if (cc.skipTokenOfType(Token.Type.COLON, isOptional = true)) {
val tt = cc.requireToken(Token.Type.ID, "type name or type expression required")
val isNullable = cc.skipTokenOfType(Token.Type.QUESTION, isOptional = true)
TypeDecl.Simple(tt.value, isNullable)
} else TypeDecl.TypeAny
return parseTypeDeclarationWithMini().first
}
// Minimal helper to parse a type annotation and simultaneously build a MiniTypeRef.
// Currently supports a simple identifier with optional nullable '?'.
private fun parseTypeDeclarationWithMini(): Pair<TypeDecl, MiniTypeRef?> {
// Only parse a type if a ':' follows; otherwise keep current behavior
if (!cc.skipTokenOfType(Token.Type.COLON, isOptional = true)) return Pair(TypeDecl.TypeAny, null)
// Parse a qualified base name: ID ('.' ID)*
val segments = mutableListOf<MiniTypeName.Segment>()
var first = true
val typeStart = cc.currentPos()
var lastEnd = typeStart
while (true) {
val idTok = if (first) cc.requireToken(Token.Type.ID, "type name or type expression required") else cc.requireToken(Token.Type.ID, "identifier expected after '.' in type")
first = false
segments += MiniTypeName.Segment(idTok.value, MiniRange(idTok.pos, idTok.pos))
lastEnd = cc.currentPos()
val dotPos = cc.savePos()
val t = cc.next()
if (t.type == Token.Type.DOT) {
// continue
continue
} else {
cc.restorePos(dotPos)
break
}
}
// Helper to build MiniTypeRef (base or generic)
fun buildBaseRef(rangeEnd: Pos, args: List<MiniTypeRef>?, nullable: Boolean): MiniTypeRef {
val base = MiniTypeName(MiniRange(typeStart, rangeEnd), segments.toList(), nullable = false)
return if (args == null || args.isEmpty()) base.copy(range = MiniRange(typeStart, rangeEnd), nullable = nullable)
else net.sergeych.lyng.miniast.MiniGenericType(MiniRange(typeStart, rangeEnd), base, args, nullable)
}
// Optional generic arguments: '<' Type (',' Type)* '>' — single-level only (no nested generics for now)
var args: MutableList<MiniTypeRef>? = null
val afterBasePos = cc.savePos()
if (cc.skipTokenOfType(Token.Type.LT, isOptional = true)) {
args = mutableListOf()
do {
// Parse argument as simple or qualified type (single level), with optional nullable '?'
val argSegs = mutableListOf<MiniTypeName.Segment>()
var argFirst = true
val argStart = cc.currentPos()
while (true) {
val idTok = if (argFirst) cc.requireToken(Token.Type.ID, "type argument name expected") else cc.requireToken(Token.Type.ID, "identifier expected after '.' in type argument")
argFirst = false
argSegs += MiniTypeName.Segment(idTok.value, MiniRange(idTok.pos, idTok.pos))
val p = cc.savePos()
val tt = cc.next()
if (tt.type == Token.Type.DOT) continue else { cc.restorePos(p); break }
}
val argNullable = cc.skipTokenOfType(Token.Type.QUESTION, isOptional = true)
val argEnd = cc.currentPos()
val argRef = MiniTypeName(MiniRange(argStart, argEnd), argSegs.toList(), nullable = argNullable)
args += argRef
val sep = cc.next()
when (sep.type) {
Token.Type.COMMA -> { /* continue */ }
Token.Type.GT -> break
else -> sep.raiseSyntax("expected ',' or '>' in generic arguments")
}
} while (true)
lastEnd = cc.currentPos()
} else {
cc.restorePos(afterBasePos)
}
// Nullable suffix after base or generic
val isNullable = cc.skipTokenOfType(Token.Type.QUESTION, isOptional = true)
val endPos = cc.currentPos()
val miniRef = buildBaseRef(if (args != null) endPos else lastEnd, args, isNullable)
// Semantic: keep simple for now, just use qualified base name with nullable flag
val qualified = segments.joinToString(".") { it.name }
val sem = TypeDecl.Simple(qualified, isNullable)
return Pair(sem, miniRef)
}
/**
@@ -871,11 +1053,27 @@ class Compiler(
* @return parsed statement or null if, for example. [id] is not among keywords
*/
private suspend fun parseKeywordStatement(id: Token): Statement? = when (id.value) {
"val" -> parseVarDeclaration(false, Visibility.Public)
"var" -> parseVarDeclaration(true, Visibility.Public)
"val" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseVarDeclaration(false, Visibility.Public)
}
"var" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseVarDeclaration(true, Visibility.Public)
}
// Ensure function declarations are recognized in all contexts (including class bodies)
"fun" -> parseFunctionDeclaration(isOpen = false, isExtern = false, isStatic = false)
"fn" -> parseFunctionDeclaration(isOpen = false, isExtern = false, isStatic = false)
"fun" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseFunctionDeclaration(isOpen = false, isExtern = false, isStatic = false)
}
"fn" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseFunctionDeclaration(isOpen = false, isExtern = false, isStatic = false)
}
// Visibility modifiers for declarations: private/protected val/var/fun/fn
"private" -> {
var k = cc.requireToken(Token.Type.ID, "declaration expected after 'private'")
@@ -913,8 +1111,16 @@ class Compiler(
"break" -> parseBreakStatement(id.pos)
"continue" -> parseContinueStatement(id.pos)
"if" -> parseIfStatement()
"class" -> parseClassDeclaration()
"enum" -> parseEnumDeclaration()
"class" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseClassDeclaration()
}
"enum" -> {
pendingDeclStart = id.pos
pendingDeclDoc = consumePendingDoc()
parseEnumDeclaration()
}
"try" -> parseTryStatement()
"throw" -> parseThrowStatement(id.pos)
"when" -> parseWhenStatement()
@@ -923,7 +1129,10 @@ class Compiler(
cc.previous()
val isExtern = cc.skipId("extern")
when {
cc.matchQualifiers("fun", "private") -> parseFunctionDeclaration(Visibility.Private, isExtern)
cc.matchQualifiers("fun", "private") -> {
pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc();
parseFunctionDeclaration(Visibility.Private, isExtern)
}
cc.matchQualifiers("fun", "private", "static") -> parseFunctionDeclaration(
Visibility.Private,
isExtern,
@@ -940,27 +1149,27 @@ class Compiler(
cc.matchQualifiers("fun", "open") -> parseFunctionDeclaration(isOpen = true, isExtern = isExtern)
cc.matchQualifiers("fn", "open") -> parseFunctionDeclaration(isOpen = true, isExtern = isExtern)
cc.matchQualifiers("fun") -> parseFunctionDeclaration(isOpen = false, isExtern = isExtern)
cc.matchQualifiers("fn") -> parseFunctionDeclaration(isOpen = false, isExtern = isExtern)
cc.matchQualifiers("fun") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseFunctionDeclaration(isOpen = false, isExtern = isExtern) }
cc.matchQualifiers("fn") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseFunctionDeclaration(isOpen = false, isExtern = isExtern) }
cc.matchQualifiers("val", "private", "static") -> parseVarDeclaration(
cc.matchQualifiers("val", "private", "static") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(
false,
Visibility.Private,
isStatic = true
)
) }
cc.matchQualifiers("val", "static") -> parseVarDeclaration(false, Visibility.Public, isStatic = true)
cc.matchQualifiers("val", "private") -> parseVarDeclaration(false, Visibility.Private)
cc.matchQualifiers("var", "static") -> parseVarDeclaration(true, Visibility.Public, isStatic = true)
cc.matchQualifiers("var", "static", "private") -> parseVarDeclaration(
cc.matchQualifiers("val", "static") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(false, Visibility.Public, isStatic = true) }
cc.matchQualifiers("val", "private") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(false, Visibility.Private) }
cc.matchQualifiers("var", "static") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(true, Visibility.Public, isStatic = true) }
cc.matchQualifiers("var", "static", "private") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(
true,
Visibility.Private,
isStatic = true
)
) }
cc.matchQualifiers("var", "private") -> parseVarDeclaration(true, Visibility.Private)
cc.matchQualifiers("val", "open") -> parseVarDeclaration(false, Visibility.Private, true)
cc.matchQualifiers("var", "open") -> parseVarDeclaration(true, Visibility.Private, true)
cc.matchQualifiers("var", "private") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(true, Visibility.Private) }
cc.matchQualifiers("val", "open") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(false, Visibility.Private, true) }
cc.matchQualifiers("var", "open") -> { pendingDeclStart = id.pos; pendingDeclDoc = consumePendingDoc(); parseVarDeclaration(true, Visibility.Private, true) }
else -> {
cc.next()
null
@@ -1283,14 +1492,18 @@ class Compiler(
pushInitScope()
// Robust body detection: peek next non-whitespace token; if it's '{', consume and parse the body
var classBodyRange: MiniRange? = null
val bodyInit: Statement? = run {
val saved = cc.savePos()
val next = cc.nextNonWhitespace()
if (next.type == Token.Type.LBRACE) {
// parse body
parseScript().also {
cc.skipTokens(Token.Type.RBRACE)
}
val bodyStart = next.pos
val st = parseScript()
val rbTok = cc.next()
if (rbTok.type != Token.Type.RBRACE) throw ScriptError(rbTok.pos, "unbalanced braces in class body")
classBodyRange = MiniRange(bodyStart, rbTok.pos)
st
} else {
// restore if no body starts here
cc.restorePos(saved)
@@ -1298,6 +1511,39 @@ class Compiler(
}
}
// Emit MiniClassDecl with collected base names; bodyRange is omitted for now
run {
val declRange = MiniRange(pendingDeclStart ?: nameToken.pos, cc.currentPos())
val bases = baseSpecs.map { it.name }
// Collect constructor fields declared as val/var in primary constructor
val ctorFields = mutableListOf<net.sergeych.lyng.miniast.MiniCtorField>()
constructorArgsDeclaration?.let { ad ->
for (p in ad.params) {
val at = p.accessType
if (at != null) {
val mutable = at == AccessType.Var
ctorFields += net.sergeych.lyng.miniast.MiniCtorField(
name = p.name,
mutable = mutable,
type = p.miniType,
nameStart = p.pos
)
}
}
}
val node = MiniClassDecl(
range = declRange,
name = nameToken.value,
bases = bases,
bodyRange = classBodyRange,
ctorFields = ctorFields,
doc = pendingDeclDoc,
nameStart = nameToken.pos
)
miniSink?.onClassDecl(node)
pendingDeclDoc = null
}
val initScope = popInitScope()
// create class
@@ -1773,6 +2019,7 @@ class Compiler(
var name = if (t.type != Token.Type.ID)
throw ScriptError(t.pos, "Expected identifier after 'fun'")
else t.value
var nameStartPos: Pos = t.pos
val annotation = lastAnnotation
val parentContext = codeContexts.last()
@@ -1785,6 +2032,7 @@ class Compiler(
if (t.type != Token.Type.ID)
throw ScriptError(t.pos, "illegal extension format: expected function name")
name = t.value
nameStartPos = t.pos
t = cc.next()
}
@@ -1798,7 +2046,36 @@ class Compiler(
"Bad function definition: expected valid argument declaration or () after 'fn ${name}'"
)
if (cc.current().type == Token.Type.COLON) parseTypeDeclaration()
// Optional return type
val returnTypeMini: MiniTypeRef? = if (cc.current().type == Token.Type.COLON) {
parseTypeDeclarationWithMini().second
} else null
// Capture doc locally to reuse even if we need to emit later
val declDocLocal = pendingDeclDoc
// Emit MiniFunDecl before body parsing (body range unknown yet)
run {
val params = argsDeclaration.params.map { p ->
MiniParam(
name = p.name,
type = p.miniType,
nameStart = p.pos
)
}
val declRange = MiniRange(pendingDeclStart ?: start, cc.currentPos())
val node = MiniFunDecl(
range = declRange,
name = name,
params = params,
returnType = returnTypeMini,
body = null,
doc = declDocLocal,
nameStart = nameStartPos
)
miniSink?.onFunDecl(node)
pendingDeclDoc = null
}
return inCodeContext(CodeContext.Function(name)) {
@@ -1894,6 +2171,27 @@ class Compiler(
NopStatement
} else
fnCreateStatement
}.also {
val bodyRange = lastParsedBlockRange
// Also emit a post-parse MiniFunDecl to be robust in case early emission was skipped by some path
val params = argsDeclaration.params.map { p ->
MiniParam(
name = p.name,
type = p.miniType,
nameStart = p.pos
)
}
val declRange = MiniRange(pendingDeclStart ?: start, cc.currentPos())
val node = MiniFunDecl(
range = declRange,
name = name,
params = params,
returnType = returnTypeMini,
body = bodyRange?.let { MiniBlock(it) },
doc = declDocLocal,
nameStart = nameStartPos
)
miniSink?.onFunDecl(node)
}
}
@@ -1912,6 +2210,10 @@ class Compiler(
val t1 = cc.next()
if (t1.type != Token.Type.RBRACE)
throw ScriptError(t1.pos, "unbalanced braces: expected block body end: }")
// Record last parsed block range and notify Mini-AST sink
val range = MiniRange(startPos, t1.pos)
lastParsedBlockRange = range
miniSink?.onBlock(MiniBlock(range))
}
}
@@ -1927,6 +2229,11 @@ class Compiler(
throw ScriptError(nameToken.pos, "Expected identifier here")
val name = nameToken.value
// Optional explicit type annotation
val varTypeMini: MiniTypeRef? = if (cc.current().type == Token.Type.COLON) {
parseTypeDeclarationWithMini().second
} else null
val eqToken = cc.next()
var setNull = false
@@ -1951,6 +2258,23 @@ class Compiler(
else parseStatement(true)
?: throw ScriptError(eqToken.pos, "Expected initializer expression")
// Emit MiniValDecl for this declaration (before execution wiring), attach doc if any
run {
val declRange = MiniRange(pendingDeclStart ?: start, cc.currentPos())
val initR = if (setNull) null else MiniRange(eqToken.pos, cc.currentPos())
val node = MiniValDecl(
range = declRange,
name = name,
mutable = isMutable,
type = varTypeMini,
initRange = initR,
doc = pendingDeclDoc,
nameStart = nameToken.pos
)
miniSink?.onValDecl(node)
pendingDeclDoc = null
}
if (isStatic) {
// find objclass instance: this is tricky: this code executes in object initializer,
// when creating instance, but we need to execute it in the class initializer which
@@ -2049,6 +2373,18 @@ class Compiler(
return Compiler(CompilerContext(parseLyng(source)), importManager).parseScript()
}
/**
* Compile [source] while streaming a Mini-AST into the provided [sink].
* When [sink] is null, behaves like [compile].
*/
suspend fun compileWithMini(source: Source, importManager: ImportProvider, sink: net.sergeych.lyng.miniast.MiniAstSink?): Script {
return Compiler(CompilerContext(parseLyng(source)), importManager, Settings(miniAstSink = sink)).parseScript()
}
/** Convenience overload to compile raw [code] with a Mini-AST [sink]. */
suspend fun compileWithMini(code: String, sink: net.sergeych.lyng.miniast.MiniAstSink?): Script =
compileWithMini(Source("<eval>", code), Script.defaultImportManager, sink)
private var lastPriority = 0
// Helpers for conservative constant folding (literal-only). Only pure, side-effect-free ops.
private fun constOf(r: ObjRef): Obj? = (r as? ConstRef)?.constValue
@@ -69,7 +69,15 @@ class CompilerContext(val tokens: List<Token>) {
throw ScriptError(currentPos(), message)
}
fun currentPos(): Pos = tokens[currentIndex].pos
fun currentPos(): Pos {
if (tokens.isEmpty()) return Pos.builtIn
val idx = when {
currentIndex < 0 -> 0
currentIndex >= tokens.size -> tokens.size - 1
else -> currentIndex
}
return tokens[idx].pos
}
/**
* If the next token is identifier `name`, skip it and return `true`.
@@ -0,0 +1,284 @@
/*
* Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/*
* Minimal binding for editor/highlighter: builds a simple scope tree from Mini-AST
* and binds identifier usages to declarations without type analysis.
*/
package net.sergeych.lyng.binding
import net.sergeych.lyng.Source
import net.sergeych.lyng.highlight.HighlightKind
import net.sergeych.lyng.highlight.SimpleLyngHighlighter
import net.sergeych.lyng.highlight.offsetOf
import net.sergeych.lyng.miniast.MiniClassDecl
import net.sergeych.lyng.miniast.MiniFunDecl
import net.sergeych.lyng.miniast.MiniScript
import net.sergeych.lyng.miniast.MiniValDecl
enum class SymbolKind { Class, Enum, Function, Val, Var, Param }
data class Symbol(
val id: Int,
val name: String,
val kind: SymbolKind,
val declStart: Int,
val declEnd: Int,
val containerId: Int?
)
data class Reference(val symbolId: Int, val start: Int, val end: Int)
data class BindingSnapshot(
val symbols: List<Symbol>,
val references: List<Reference>
)
/**
* Very small binder that:
* - Registers symbols for top-level decls, function params, and local vals/vars inside function bodies.
* - Binds identifier tokens to the nearest matching symbol by lexical scope (function locals/params first, then top-level).
*/
object Binder {
fun bind(text: String, mini: MiniScript): BindingSnapshot {
val source = Source("<snippet>", text)
val highlighter = SimpleLyngHighlighter()
val spans = highlighter.highlight(text)
// 1) Collect symbols
val symbols = ArrayList<Symbol>()
var nextId = 1
// Helper to convert Pos to offsets
fun nameOffsets(startPos: net.sergeych.lyng.Pos, name: String): Pair<Int, Int> {
val s = source.offsetOf(startPos)
return s to (s + name.length)
}
// Class scopes to support fields and method resolution order
data class ClassScope(
val symId: Int,
val bodyStart: Int,
val bodyEnd: Int,
val fields: MutableList<Int>
)
val classes = ArrayList<ClassScope>()
// Map of function id to its local symbol ids and owning class (if method)
data class FnScope(
val id: Int,
val rangeStart: Int,
val rangeEnd: Int,
val locals: MutableList<Int>,
val classOwnerId: Int?
)
val functions = ArrayList<FnScope>()
// Index top-level for quick lookups
val topLevelByName = HashMap<String, MutableList<Int>>()
// Helper to find class that contains an offset (by body range)
fun classContaining(offset: Int): ClassScope? =
classes.asSequence()
.filter { it.bodyEnd > it.bodyStart && offset >= it.bodyStart && offset <= it.bodyEnd }
.maxByOrNull { it.bodyEnd - it.bodyStart }
// First pass (classes only): register classes so we can attach methods/fields
for (d in mini.declarations) if (d is MiniClassDecl) {
val (s, e) = nameOffsets(d.nameStart, d.name)
val sym = Symbol(nextId++, d.name, SymbolKind.Class, s, e, containerId = null)
symbols += sym
topLevelByName.getOrPut(d.name) { mutableListOf() }.add(sym.id)
// Prefer explicit body range; otherwise use the whole class declaration range
val bodyStart = d.bodyRange?.start?.let { source.offsetOf(it) } ?: source.offsetOf(d.range.start)
val bodyEnd = d.bodyRange?.end?.let { source.offsetOf(it) } ?: source.offsetOf(d.range.end)
classes += ClassScope(sym.id, bodyStart, bodyEnd, mutableListOf())
// Constructor fields (val/var in primary ctor)
for (cf in d.ctorFields) {
val fs = source.offsetOf(cf.nameStart)
val fe = fs + cf.name.length
val kind = if (cf.mutable) SymbolKind.Var else SymbolKind.Val
val fieldSym = Symbol(nextId++, cf.name, kind, fs, fe, containerId = sym.id)
symbols += fieldSym
classes.last().fields += fieldSym.id
}
}
// Second pass: functions and top-level/class vals/vars
for (d in mini.declarations) {
when (d) {
is MiniClassDecl -> { /* already processed in first pass */ }
is MiniFunDecl -> {
val (s, e) = nameOffsets(d.nameStart, d.name)
val ownerClass = classContaining(s)
val sym = Symbol(nextId++, d.name, SymbolKind.Function, s, e, containerId = ownerClass?.symId)
symbols += sym
topLevelByName.getOrPut(d.name) { mutableListOf() }.add(sym.id)
// Determine body range if present; otherwise, derive a conservative end at decl range end
val bodyStart = d.body?.range?.start?.let { source.offsetOf(it) } ?: e
val bodyEnd = d.body?.range?.end?.let { source.offsetOf(it) } ?: e
val fnScope = FnScope(sym.id, bodyStart, bodyEnd, mutableListOf(), ownerClass?.symId)
// Params
for (p in d.params) {
val ps = source.offsetOf(p.nameStart)
val pe = ps + p.name.length
val pk = SymbolKind.Param
val paramSym = Symbol(nextId++, p.name, pk, ps, pe, containerId = sym.id)
fnScope.locals += paramSym.id
symbols += paramSym
}
functions += fnScope
}
is MiniValDecl -> {
val (s, e) = nameOffsets(d.nameStart, d.name)
val kind = if (d.mutable) SymbolKind.Var else SymbolKind.Val
val ownerClass = classContaining(s)
if (ownerClass != null) {
// class field
val fieldSym = Symbol(nextId++, d.name, kind, s, e, containerId = ownerClass.symId)
symbols += fieldSym
ownerClass.fields += fieldSym.id
} else {
val sym = Symbol(nextId++, d.name, kind, s, e, containerId = null)
symbols += sym
topLevelByName.getOrPut(d.name) { mutableListOf() }.add(sym.id)
}
}
}
}
// Inject stdlib symbols into the top-level when imported, so calls like `filter {}` bind semantically.
run {
val importedModules = mini.imports.map { it.segments.joinToString(".") { s -> s.name } }
val hasStdlib = importedModules.any { it == "lyng.stdlib" || it.startsWith("lyng.stdlib.") }
if (hasStdlib) {
val stdFns = listOf(
"filter", "map", "flatMap", "reduce", "fold", "take", "drop", "sorted",
"groupBy", "count", "any", "all", "first", "last", "sum", "joinToString",
// iterator trio often used implicitly/explicitly
"iterator", "hasNext", "next"
)
for (name in stdFns) {
val sym = Symbol(nextId++, name, SymbolKind.Function, 0, name.length, containerId = null)
symbols += sym
topLevelByName.getOrPut(name) { mutableListOf() }.add(sym.id)
}
}
}
// Second pass: attach local val/var declarations that fall inside any function body
for (d in mini.declarations) if (d is MiniValDecl) {
val (s, e) = nameOffsets(d.nameStart, d.name)
// Find containing function by body range that includes this declaration name
val containerFn = functions.asSequence()
.filter { it.rangeEnd > it.rangeStart && s >= it.rangeStart && s <= it.rangeEnd }
.maxByOrNull { it.rangeEnd - it.rangeStart }
if (containerFn != null) {
val fnSymId = containerFn.id
val kind = if (d.mutable) SymbolKind.Var else SymbolKind.Val
val localSym = Symbol(nextId++, d.name, kind, s, e, containerId = fnSymId)
symbols += localSym
containerFn.locals += localSym.id
}
}
// Build name -> symbol ids index per function (locals+params) and per class (fields) for faster resolution
data class Idx(val byName: Map<String, List<Int>>)
fun buildIndex(ids: List<Int>): Idx {
val map = HashMap<String, MutableList<Int>>()
for (id in ids) {
val s = symbols.first { it.id == id }
map.getOrPut(s.name) { mutableListOf() }.add(id)
}
return Idx(map)
}
val fnLocalIndex = HashMap<Int, Idx>() // key: function symbol id
for (fn in functions) fnLocalIndex[fn.id] = buildIndex(fn.locals)
val classFieldIndex = HashMap<Int, Idx>() // key: class symbol id
for (cls in classes) classFieldIndex[cls.symId] = buildIndex(cls.fields)
// Helper to find nearest match among candidates: pick the declaration with the smallest (usageStart - declStart) >= 0
fun chooseNearest(candidates: List<Int>, usageStart: Int): Int? {
var best: Int? = null
var bestDist = Int.MAX_VALUE
for (id in candidates) {
val s = symbols.first { it.id == id }
val dist = usageStart - s.declStart
if (dist >= 0 && dist < bestDist) {
bestDist = dist; best = id
}
}
return best
}
// 2) Bind identifier usages to symbols
val references = ArrayList<Reference>()
for (span in spans) {
if (span.kind != HighlightKind.Identifier) continue
val start = span.range.start
val end = span.range.endExclusive
// Skip if this range equals a known declaration identifier range; those are styled separately
val isDecl = symbols.any { it.declStart == start && it.declEnd == end }
if (isDecl) continue
val name = text.substring(start, end)
// Prefer function-local matches: find the function whose body contains the usage
val inFn = functions.asSequence()
.filter { it.rangeEnd > it.rangeStart && start >= it.rangeStart && start <= it.rangeEnd }
.maxByOrNull { it.rangeEnd - it.rangeStart }
if (inFn != null) {
val idx = fnLocalIndex[inFn.id]
val locIds = idx?.byName?.get(name)
val hit = if (!locIds.isNullOrEmpty()) chooseNearest(locIds, start) else null
if (hit != null) {
references += Reference(hit, start, end)
continue
}
// If function belongs to a class, check class fields next
val ownerClassId = inFn.classOwnerId
if (ownerClassId != null) {
val cidx = classFieldIndex[ownerClassId]
val cfIds = cidx?.byName?.get(name)
val cHit = if (!cfIds.isNullOrEmpty()) chooseNearest(cfIds, start) else null
if (cHit != null) {
references += Reference(cHit, start, end)
continue
}
}
}
// Else try top-level by name
val topIds = topLevelByName[name]
val topHit = if (!topIds.isNullOrEmpty()) chooseNearest(topIds, start) else null
if (topHit != null) {
references += Reference(topHit, start, end)
continue
}
// Fallback: choose the nearest declared symbol with this name regardless of scope (best effort)
val anyIds = symbols.filter { it.name == name }.map { it.id }
val anyHit = if (anyIds.isNotEmpty()) chooseNearest(anyIds, start) else null
if (anyHit != null) references += Reference(anyHit, start, end)
}
return BindingSnapshot(symbols, references)
}
}
@@ -37,7 +37,16 @@ fun Source.offsetOf(pos: Pos): Int {
private val reservedIdKeywords = setOf("constructor", "property")
// Fallback textual keywords that might come from the lexer as ID in some contexts (e.g., snippets)
private val fallbackKeywordIds = setOf("and", "or", "not")
private val fallbackKeywordIds = setOf(
// boolean operators
"and", "or", "not",
// declarations & modifiers
"fun", "fn", "class", "enum", "val", "var", "import", "package",
"private", "protected", "static", "open", "extern",
// control flow and misc
"if", "else", "when", "while", "do", "for", "try", "catch", "finally",
"throw", "return", "break", "continue", "this", "null", "true", "false"
)
/** Maps lexer token type (and sometimes value) to a [HighlightKind]. */
private fun kindOf(type: Type, value: String): HighlightKind? = when (type) {
@@ -135,6 +144,11 @@ class SimpleLyngHighlighter : LyngHighlighter {
val range = when (t.type) {
Type.STRING, Type.STRING2 -> adjustQuoteSpan(start, '"')
Type.CHAR -> adjustQuoteSpan(start, '\'')
Type.HEX -> {
// Parser returns HEX token value without the leading "0x"; include it in highlight span
val end = (start + 2 + t.value.length).coerceAtMost(text.length)
TextRange(start, end)
}
else -> TextRange(start, (start + t.value.length).coerceAtMost(text.length))
}
if (range.endExclusive > range.start) raw += HighlightSpan(range, k)
@@ -0,0 +1,247 @@
/*
* Copyright 2025 Sergey S. Chernov real.sergeych@gmail.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/*
* Minimal, optional AST-like structures captured during parsing.
*
* These classes are lightweight and focused on editor features:
* - highlighting (precise ranges for ids/types)
* - documentation (raw doc text + summary)
* - basic autocomplete (visible names per scope)
*
* They are populated only when an optional MiniAstSink is provided to the compiler.
*/
package net.sergeych.lyng.miniast
import net.sergeych.lyng.Pos
// Ranges reuse existing Pos to stay consistent with compiler diagnostics
data class MiniRange(val start: Pos, val end: Pos)
// Simple documentation payload: raw text and a derived summary (first non-empty line)
data class MiniDoc(
val range: MiniRange,
val raw: String,
val summary: String?,
val tags: Map<String, List<String>> = emptyMap()
)
sealed interface MiniNode { val range: MiniRange }
// Identifier roles we can confidently assign at parse time without binding
enum class IdRole {
DeclFun, DeclClass, DeclVal, DeclVar, Param, Label, TypeName, Ref
}
// Type references (syntax-level, position rich)
sealed interface MiniTypeRef : MiniNode
data class MiniTypeName(
override val range: MiniRange,
val segments: List<Segment>,
val nullable: Boolean
) : MiniTypeRef {
data class Segment(val name: String, val range: MiniRange)
}
data class MiniGenericType(
override val range: MiniRange,
val base: MiniTypeRef,
val args: List<MiniTypeRef>,
val nullable: Boolean
) : MiniTypeRef
data class MiniFunctionType(
override val range: MiniRange,
val receiver: MiniTypeRef?,
val params: List<MiniTypeRef>,
val returnType: MiniTypeRef,
val nullable: Boolean
) : MiniTypeRef
data class MiniTypeVar(
override val range: MiniRange,
val name: String,
val nullable: Boolean
) : MiniTypeRef
// Script and declarations (lean subset; can be extended later)
sealed interface MiniDecl : MiniNode {
val name: String
val doc: MiniDoc?
// Start position of the declaration name identifier in source; end can be derived as start + name.length
val nameStart: Pos
}
data class MiniScript(
override val range: MiniRange,
val declarations: MutableList<MiniDecl> = mutableListOf(),
val imports: MutableList<MiniImport> = mutableListOf(),
val statements: MutableList<MiniStmt> = mutableListOf()
) : MiniNode
data class MiniParam(
val name: String,
val type: MiniTypeRef?,
// Start position of parameter name in source
val nameStart: Pos
)
data class MiniFunDecl(
override val range: MiniRange,
override val name: String,
val params: List<MiniParam>,
val returnType: MiniTypeRef?,
val body: MiniBlock?,
override val doc: MiniDoc?,
override val nameStart: Pos
) : MiniDecl
data class MiniValDecl(
override val range: MiniRange,
override val name: String,
val mutable: Boolean,
val type: MiniTypeRef?,
val initRange: MiniRange?,
override val doc: MiniDoc?,
override val nameStart: Pos
) : MiniDecl
data class MiniClassDecl(
override val range: MiniRange,
override val name: String,
val bases: List<String>,
val bodyRange: MiniRange?,
val ctorFields: List<MiniCtorField> = emptyList(),
val classFields: List<MiniCtorField> = emptyList(),
override val doc: MiniDoc?,
override val nameStart: Pos
) : MiniDecl
data class MiniCtorField(
val name: String,
val mutable: Boolean,
val type: MiniTypeRef?,
val nameStart: Pos
)
sealed interface MiniStmt : MiniNode
data class MiniBlock(
override val range: MiniRange,
val locals: MutableList<String> = mutableListOf()
) : MiniStmt
data class MiniIdentifier(
override val range: MiniRange,
val name: String,
val role: IdRole
) : MiniNode
// Streaming sink to collect mini-AST during parsing. Implementations may assemble a tree or process events.
interface MiniAstSink {
fun onScriptStart(start: Pos) {}
fun onScriptEnd(end: Pos, script: MiniScript) {}
fun onDocCandidate(doc: MiniDoc) {}
fun onImport(node: MiniImport) {}
fun onFunDecl(node: MiniFunDecl) {}
fun onValDecl(node: MiniValDecl) {}
fun onClassDecl(node: MiniClassDecl) {}
fun onBlock(node: MiniBlock) {}
fun onIdentifier(node: MiniIdentifier) {}
}
// Tracer interface used by the compiler's type parser to simultaneously build
// a MiniTypeRef (syntax-level) while the semantic TypeDecl is produced as usual.
// Keep it extremely small for now: the current parser supports only simple
// identifiers with optional nullable suffix. We can extend callbacks later for
// generics and function types without breaking callers.
interface MiniTypeTrace {
/**
* Report a simple (possibly qualified in the future) type name.
* @param segments ordered list of name parts with precise ranges
* @param nullable whether the trailing `?` was present
*/
fun onSimpleName(segments: List<MiniTypeName.Segment>, nullable: Boolean)
}
// A simple builder that assembles a MiniScript tree from sink callbacks.
class MiniAstBuilder : MiniAstSink {
private var currentScript: MiniScript? = null
private val blocks = ArrayDeque<MiniBlock>()
private var lastDoc: MiniDoc? = null
private var scriptDepth: Int = 0
fun build(): MiniScript? = currentScript
override fun onScriptStart(start: Pos) {
if (scriptDepth == 0) {
currentScript = MiniScript(MiniRange(start, start))
}
scriptDepth++
}
override fun onScriptEnd(end: Pos, script: MiniScript) {
scriptDepth = (scriptDepth - 1).coerceAtLeast(0)
if (scriptDepth == 0) {
// finalize root range only when closing the outermost script
currentScript = currentScript?.copy(range = MiniRange(currentScript!!.range.start, end))
}
}
override fun onDocCandidate(doc: MiniDoc) {
lastDoc = doc
}
override fun onImport(node: MiniImport) {
currentScript?.imports?.add(node)
}
override fun onFunDecl(node: MiniFunDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
lastDoc = null
}
override fun onValDecl(node: MiniValDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
lastDoc = null
}
override fun onClassDecl(node: MiniClassDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
lastDoc = null
}
override fun onBlock(node: MiniBlock) {
blocks.addLast(node)
}
}
// Import statement representation for highlighting and docs
data class MiniImport(
override val range: MiniRange,
val segments: List<Segment>
) : MiniNode {
data class Segment(val name: String, val range: MiniRange)
}
@@ -43,11 +43,18 @@ class ObjFlowBuilder(val output: SendChannel<Obj>) : Obj() {
if (!channel.isClosedForSend)
channel.send(data)
else
// Flow consumer is no longer collecting; signal producer to stop
throw ScriptFlowIsNoMoreCollected()
} catch (x: Exception) {
if (x !is CancellationException)
x.printStackTrace()
throw ScriptFlowIsNoMoreCollected()
// Any failure to send (including closed channel) should gracefully stop the producer.
// Do not print stack traces here to keep test output clean on JVM.
if (x is CancellationException) {
// Cancellation is a normal control-flow event
throw ScriptFlowIsNoMoreCollected()
} else {
// Treat other send failures as normal flow termination as well
throw ScriptFlowIsNoMoreCollected()
}
}
ObjVoid
}
@@ -63,10 +70,10 @@ private fun createLyngFlowInput(scope: Scope, producer: Statement): ReceiveChann
try {
producer.execute(builderScope)
} catch (x: ScriptFlowIsNoMoreCollected) {
x.printStackTrace()
// premature flow closing, OK
} catch (x: Exception) {
x.printStackTrace()
// Suppress stack traces in background producer to avoid noisy stderr during tests.
// If needed, consider routing to a logger in the future.
}
channel.close()
}