SergeychWorks/lyng
1
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases 21 Wiki Activity

Added support for extern declarations and enhanced .lyng.d merging

Browse Source 
- Implemented `extern` support for functions, classes, objects, enums, and properties in the `MiniAST`.
- Updated `MiniAST` to include `isExtern` field for applicable nodes.
- Enabled merging of `.lyng.d` declaration files into main `.lyng` scripts.
- Adjusted tests to validate `extern` behavior and documentation handling.
- Minor fixes to parser logic for improved robustness.
This commit is contained in:
sergeych committed 2026-01-06 17:05:03 +01:00
1 parent 44675b976d
commit fdc044d1e0
8 files changed
+310 -95
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lynglib/src/commonMain/kotlin/net/sergeych/lyng/CodeContext.kt
+1 -1
View File
@@ -20,7 +20,7 @@ package net.sergeych.lyng
sealed class CodeContext {
class Module(@Suppress("unused") val packageName: String?): CodeContext()
class Function(val name: String): CodeContext()
class ClassBody(val name: String): CodeContext() {
class ClassBody(val name: String, val isExtern: Boolean = false): CodeContext() {
val pendingInitializations = mutableMapOf<String, Pos>()
}
}
lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt
+127 -59
View File
@@ -1327,7 +1327,7 @@ class Compiler(
"private", "protected", "static", "abstract", "closed", "override", "extern", "open" -> {
modifiers.add(currentToken.value)
val next = cc.peekNextNonWhitespace()
if (next.type == Token.Type.ID) {
if (next.type == Token.Type.ID || next.type == Token.Type.OBJECT) {
currentToken = cc.next()
} else {
break
@@ -1367,32 +1367,50 @@ class Compiler(
throw ScriptError(currentToken.pos, "modifier abstract at top level is only allowed for classes")
return when (currentToken.value) {
"val" -> parseVarDeclaration(false, visibility, isAbstract, isClosed, isOverride, isStatic)
"var" -> parseVarDeclaration(true, visibility, isAbstract, isClosed, isOverride, isStatic)
"val" -> parseVarDeclaration(false, visibility, isAbstract, isClosed, isOverride, isStatic, isExtern)
"var" -> parseVarDeclaration(true, visibility, isAbstract, isClosed, isOverride, isStatic, isExtern)
"fun", "fn" -> parseFunctionDeclaration(visibility, isAbstract, isClosed, isOverride, isExtern, isStatic)
"class" -> {
if (isStatic || isClosed || isOverride || isExtern)
throw ScriptError(currentToken.pos, "unsupported modifiers for class: ${modifiers.joinToString(" ")}")
parseClassDeclaration(isAbstract)
if (isStatic || isClosed || isOverride)
throw ScriptError(
currentToken.pos,
"unsupported modifiers for class: ${modifiers.joinToString(" ")}"
)
parseClassDeclaration(isAbstract, isExtern)
}
"object" -> {
if (isStatic || isClosed || isOverride || isExtern || isAbstract)
throw ScriptError(currentToken.pos, "unsupported modifiers for object: ${modifiers.joinToString(" ")}")
parseObjectDeclaration()
if (isStatic || isClosed || isOverride || isAbstract)
throw ScriptError(
currentToken.pos,
"unsupported modifiers for object: ${modifiers.joinToString(" ")}"
)
parseObjectDeclaration(isExtern)
}
"interface" -> {
if (isStatic || isClosed || isOverride || isExtern || isAbstract)
if (isStatic || isClosed || isOverride || isAbstract)
throw ScriptError(
currentToken.pos,
"unsupported modifiers for interface: ${modifiers.joinToString(" ")}"
)
// interface is synonym for abstract class
parseClassDeclaration(isAbstract = true)
parseClassDeclaration(isAbstract = true, isExtern = isExtern)
}
else -> throw ScriptError(currentToken.pos, "expected declaration after modifiers, found ${currentToken.value}")
"enum" -> {
if (isStatic || isClosed || isOverride || isAbstract)
throw ScriptError(
currentToken.pos,
"unsupported modifiers for enum: ${modifiers.joinToString(" ")}"
)
parseEnumDeclaration(isExtern)
}
else -> throw ScriptError(
currentToken.pos,
"expected declaration after modifiers, found ${currentToken.value}"
)
}
}
@@ -1401,7 +1419,7 @@ class Compiler(
* @return parsed statement or null if, for example. [id] is not among keywords
*/
private suspend fun parseKeywordStatement(id: Token): Statement? = when (id.value) {
"abstract", "closed", "override", "extern", "private", "protected", "static" -> {
"abstract", "closed", "override", "extern", "private", "protected", "static", "open" -> {
parseDeclarationWithModifiers(id)
}
@@ -1835,7 +1853,7 @@ class Compiler(
}
}
private fun parseEnumDeclaration(): Statement {
private fun parseEnumDeclaration(isExtern: Boolean = false): Statement {
val nameToken = cc.requireToken(Token.Type.ID)
val startPos = pendingDeclStart ?: nameToken.pos
val doc = pendingDeclDoc ?: consumePendingDoc()
@@ -1873,7 +1891,8 @@ class Compiler(
name = nameToken.value,
entries = names,
doc = doc,
nameStart = nameToken.pos
nameStart = nameToken.pos,
isExtern = isExtern
)
)
@@ -1884,7 +1903,7 @@ class Compiler(
}
}
private suspend fun parseObjectDeclaration(): Statement {
private suspend fun parseObjectDeclaration(isExtern: Boolean = false): Statement {
val next = cc.peekNextNonWhitespace()
val nameToken = if (next.type == Token.Type.ID) cc.requireToken(Token.Type.ID) else null
@@ -1916,10 +1935,24 @@ class Compiler(
// Robust body detection
var classBodyRange: MiniRange? = null
val bodyInit: Statement? = inCodeContext(CodeContext.ClassBody(className)) {
val bodyInit: Statement? = inCodeContext(CodeContext.ClassBody(className, isExtern = isExtern)) {
val saved = cc.savePos()
val nextBody = cc.nextNonWhitespace()
if (nextBody.type == Token.Type.LBRACE) {
// Emit MiniClassDecl before body parsing to track members via enter/exit
run {
val node = MiniClassDecl(
range = MiniRange(startPos, cc.currentPos()),
name = className,
bases = baseSpecs.map { it.name },
bodyRange = null,
doc = doc,
nameStart = nameToken?.pos ?: startPos,
isObject = true,
isExtern = isExtern
)
miniSink?.onEnterClass(node)
}
val bodyStart = nextBody.pos
val st = withLocalNames(emptySet()) {
parseScript()
@@ -1927,8 +1960,23 @@ class Compiler(
val rbTok = cc.next()
if (rbTok.type != Token.Type.RBRACE) throw ScriptError(rbTok.pos, "unbalanced braces in object body")
classBodyRange = MiniRange(bodyStart, rbTok.pos)
miniSink?.onExitClass(rbTok.pos)
st
} else {
// No body, but still emit the class
run {
val node = MiniClassDecl(
range = MiniRange(startPos, cc.currentPos()),
name = className,
bases = baseSpecs.map { it.name },
bodyRange = null,
doc = doc,
nameStart = nameToken?.pos ?: startPos,
isObject = true,
isExtern = isExtern
)
miniSink?.onClassDecl(node)
}
cc.restorePos(saved)
null
}
@@ -1965,13 +2013,13 @@ class Compiler(
}
}
private suspend fun parseClassDeclaration(isAbstract: Boolean = false): Statement {
private suspend fun parseClassDeclaration(isAbstract: Boolean = false, isExtern: Boolean = false): Statement {
val nameToken = cc.requireToken(Token.Type.ID)
val startPos = pendingDeclStart ?: nameToken.pos
val doc = pendingDeclDoc ?: consumePendingDoc()
pendingDeclDoc = null
pendingDeclStart = null
return inCodeContext(CodeContext.ClassBody(nameToken.value)) {
return inCodeContext(CodeContext.ClassBody(nameToken.value, isExtern = isExtern)) {
val constructorArgsDeclaration =
if (cc.skipTokenOfType(Token.Type.LPAREN, isOptional = true))
parseArgsDeclaration(isClassDeclaration = true)
@@ -2009,28 +2057,7 @@ class Compiler(
val bodyInit: Statement? = run {
val saved = cc.savePos()
val next = cc.nextNonWhitespace()
if (next.type == Token.Type.LBRACE) {
// parse body
val bodyStart = next.pos
val st = withLocalNames(constructorArgsDeclaration?.params?.map { it.name }?.toSet() ?: emptySet()) {
parseScript()
}
val rbTok = cc.next()
if (rbTok.type != Token.Type.RBRACE) throw ScriptError(rbTok.pos, "unbalanced braces in class body")
classBodyRange = MiniRange(bodyStart, rbTok.pos)
st
} else {
// restore if no body starts here
cc.restorePos(saved)
null
}
}
// Emit MiniClassDecl with collected base names; bodyRange is omitted for now
run {
val declRange = MiniRange(startPos, cc.currentPos())
val bases = baseSpecs.map { it.name }
// Collect constructor fields declared in primary constructor
val ctorFields = mutableListOf<MiniCtorField>()
constructorArgsDeclaration?.let { ad ->
for (p in ad.params) {
@@ -2044,16 +2071,51 @@ class Compiler(
)
}
}
val node = MiniClassDecl(
range = declRange,
name = nameToken.value,
bases = bases,
bodyRange = classBodyRange,
ctorFields = ctorFields,
doc = doc,
nameStart = nameToken.pos
)
miniSink?.onClassDecl(node)
if (next.type == Token.Type.LBRACE) {
// Emit MiniClassDecl before body parsing to track members via enter/exit
run {
val node = MiniClassDecl(
range = MiniRange(startPos, cc.currentPos()),
name = nameToken.value,
bases = baseSpecs.map { it.name },
bodyRange = null,
ctorFields = ctorFields,
doc = doc,
nameStart = nameToken.pos,
isExtern = isExtern
)
miniSink?.onEnterClass(node)
}
// parse body
val bodyStart = next.pos
val st = withLocalNames(constructorArgsDeclaration?.params?.map { it.name }?.toSet() ?: emptySet()) {
parseScript()
}
val rbTok = cc.next()
if (rbTok.type != Token.Type.RBRACE) throw ScriptError(rbTok.pos, "unbalanced braces in class body")
classBodyRange = MiniRange(bodyStart, rbTok.pos)
miniSink?.onExitClass(rbTok.pos)
st
} else {
// No body, but still emit the class
run {
val node = MiniClassDecl(
range = MiniRange(startPos, cc.currentPos()),
name = nameToken.value,
bases = baseSpecs.map { it.name },
bodyRange = null,
ctorFields = ctorFields,
doc = doc,
nameStart = nameToken.pos,
isExtern = isExtern
)
miniSink?.onClassDecl(node)
}
// restore if no body starts here
cc.restorePos(saved)
null
}
}
val initScope = popInitScope()
@@ -2593,6 +2655,7 @@ class Compiler(
isExtern: Boolean = false,
isStatic: Boolean = false,
): Statement {
val actualExtern = isExtern || (codeContexts.lastOrNull() as? CodeContext.ClassBody)?.isExtern == true
var t = cc.next()
val start = t.pos
var extTypeName: String? = null
@@ -2669,7 +2732,8 @@ class Compiler(
body = null,
doc = declDocLocal,
nameStart = nameStartPos,
receiver = receiverMini
receiver = receiverMini,
isExtern = actualExtern
)
miniSink?.onFunDecl(node)
pendingDeclDoc = null
@@ -2684,7 +2748,7 @@ class Compiler(
// Parse function body while tracking declared locals to compute precise capacity hints
currentLocalDeclCount
localDeclCountStack.add(0)
val fnStatements = if (isExtern)
val fnStatements = if (actualExtern)
statement { raiseError("extern function not provided: $name") }
else if (isAbstract || isDelegated) {
null
@@ -2906,8 +2970,10 @@ class Compiler(
isAbstract: Boolean = false,
isClosed: Boolean = false,
isOverride: Boolean = false,
isStatic: Boolean = false
isStatic: Boolean = false,
isExtern: Boolean = false
): Statement {
val actualExtern = isExtern || (codeContexts.lastOrNull() as? CodeContext.ClassBody)?.isExtern == true
val nextToken = cc.next()
val start = nextToken.pos
@@ -2929,7 +2995,8 @@ class Compiler(
type = null,
initRange = null,
doc = pendingDeclDoc,
nameStart = namePos
nameStart = namePos,
isExtern = actualExtern
)
miniSink?.onValDecl(node)
}
@@ -3018,10 +3085,10 @@ class Compiler(
// Register the local name at compile time so that subsequent identifiers can be emitted as fast locals
if (!isStatic) declareLocalName(name)
val isDelegate = if (isAbstract) {
val isDelegate = if (isAbstract || actualExtern) {
if (!isProperty && (eqToken.type == Token.Type.ASSIGN || eqToken.type == Token.Type.BY))
throw ScriptError(eqToken.pos, "abstract variable $name cannot have an initializer or delegate")
// Abstract variables don't have initializers
throw ScriptError(eqToken.pos, "${if (isAbstract) "abstract" else "extern"} variable $name cannot have an initializer or delegate")
// Abstract or extern variables don't have initializers
cc.restorePos(markBeforeEq)
cc.skipWsTokens()
setNull = true
@@ -3063,7 +3130,8 @@ class Compiler(
initRange = initR,
doc = pendingDeclDoc,
nameStart = nameStartPos,
receiver = receiverMini
receiver = receiverMini,
isExtern = actualExtern
)
miniSink?.onValDecl(node)
pendingDeclDoc = null
lynglib/src/commonMain/kotlin/net/sergeych/lyng/CompilerContext.kt
+2 -1
View File
@@ -151,10 +151,11 @@ class CompilerContext(val tokens: List<Token>) {
* @return next non-whitespace token without extracting it from tokens list
*/
fun peekNextNonWhitespace(): Token {
val saved = savePos()
while (true) {
val t = next()
if (t.type !in wstokens) {
previous()
restorePos(saved)
return t
}
}
lynglib/src/commonMain/kotlin/net/sergeych/lyng/miniast/MiniAst.kt
+77 -9
View File
@@ -86,6 +86,7 @@ sealed interface MiniDecl : MiniNode {
val doc: MiniDoc?
// Start position of the declaration name identifier in source; end can be derived as start + name.length
val nameStart: Pos
val isExtern: Boolean
}
data class MiniScript(
@@ -110,7 +111,8 @@ data class MiniFunDecl(
val body: MiniBlock?,
override val doc: MiniDoc?,
override val nameStart: Pos,
val receiver: MiniTypeRef? = null
val receiver: MiniTypeRef? = null,
override val isExtern: Boolean = false
) : MiniDecl
data class MiniValDecl(
@@ -121,7 +123,8 @@ data class MiniValDecl(
val initRange: MiniRange?,
override val doc: MiniDoc?,
override val nameStart: Pos,
val receiver: MiniTypeRef? = null
val receiver: MiniTypeRef? = null,
override val isExtern: Boolean = false
) : MiniDecl
data class MiniClassDecl(
@@ -134,7 +137,9 @@ data class MiniClassDecl(
override val doc: MiniDoc?,
override val nameStart: Pos,
// Built-in extension: list of member declarations (functions and fields)
val members: List<MiniMemberDecl> = emptyList()
val members: List<MiniMemberDecl> = emptyList(),
override val isExtern: Boolean = false,
val isObject: Boolean = false
) : MiniDecl
data class MiniEnumDecl(
@@ -142,7 +147,8 @@ data class MiniEnumDecl(
override val name: String,
val entries: List<String>,
override val doc: MiniDoc?,
override val nameStart: Pos
override val nameStart: Pos,
override val isExtern: Boolean = false
) : MiniDecl
data class MiniCtorField(
@@ -171,6 +177,7 @@ sealed interface MiniMemberDecl : MiniNode {
val doc: MiniDoc?
val nameStart: Pos
val isStatic: Boolean
val isExtern: Boolean
}
data class MiniMemberFunDecl(
@@ -181,6 +188,7 @@ data class MiniMemberFunDecl(
override val doc: MiniDoc?,
override val nameStart: Pos,
override val isStatic: Boolean = false,
override val isExtern: Boolean = false,
) : MiniMemberDecl
data class MiniMemberValDecl(
@@ -191,6 +199,7 @@ data class MiniMemberValDecl(
override val doc: MiniDoc?,
override val nameStart: Pos,
override val isStatic: Boolean = false,
override val isExtern: Boolean = false,
) : MiniMemberDecl
data class MiniInitDecl(
@@ -200,6 +209,7 @@ data class MiniInitDecl(
override val name: String get() = "init"
override val doc: MiniDoc? get() = null
override val isStatic: Boolean get() = false
override val isExtern: Boolean get() = false
}
// Streaming sink to collect mini-AST during parsing. Implementations may assemble a tree or process events.
@@ -209,6 +219,9 @@ interface MiniAstSink {
fun onDocCandidate(doc: MiniDoc) {}
fun onEnterClass(node: MiniClassDecl) {}
fun onExitClass(end: Pos) {}
fun onImport(node: MiniImport) {}
fun onFunDecl(node: MiniFunDecl) {}
fun onValDecl(node: MiniValDecl) {}
@@ -238,6 +251,7 @@ interface MiniTypeTrace {
class MiniAstBuilder : MiniAstSink {
private var currentScript: MiniScript? = null
private val blocks = ArrayDeque<MiniBlock>()
private val classStack = ArrayDeque<MiniClassDecl>()
private var lastDoc: MiniDoc? = null
private var scriptDepth: Int = 0
@@ -262,26 +276,80 @@ class MiniAstBuilder : MiniAstSink {
lastDoc = doc
}
override fun onEnterClass(node: MiniClassDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
classStack.addLast(attach)
lastDoc = null
}
override fun onExitClass(end: Pos) {
val finished = classStack.removeLastOrNull()
if (finished != null) {
val updated = finished.copy(range = MiniRange(finished.range.start, end))
// Always add to top-level for now to ensure visibility in light engine
currentScript?.declarations?.add(updated)
}
}
override fun onImport(node: MiniImport) {
currentScript?.imports?.add(node)
}
override fun onFunDecl(node: MiniFunDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
val currentClass = classStack.lastOrNull()
if (currentClass != null) {
// Convert MiniFunDecl to MiniMemberFunDecl for inclusion in members
val member = MiniMemberFunDecl(
range = attach.range,
name = attach.name,
params = attach.params,
returnType = attach.returnType,
doc = attach.doc,
nameStart = attach.nameStart,
isStatic = false, // TODO: track static if needed
isExtern = attach.isExtern
)
// Need to update the class in the stack since it's immutable-ish (data class)
classStack.removeLast()
classStack.addLast(currentClass.copy(members = currentClass.members + member))
} else {
currentScript?.declarations?.add(attach)
}
lastDoc = null
}
override fun onValDecl(node: MiniValDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
val currentClass = classStack.lastOrNull()
if (currentClass != null) {
val member = MiniMemberValDecl(
range = attach.range,
name = attach.name,
mutable = attach.mutable,
type = attach.type,
doc = attach.doc,
nameStart = attach.nameStart,
isStatic = false, // TODO: track static if needed
isExtern = attach.isExtern
)
classStack.removeLast()
classStack.addLast(currentClass.copy(members = currentClass.members + member))
} else {
currentScript?.declarations?.add(attach)
}
lastDoc = null
}
override fun onClassDecl(node: MiniClassDecl) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
lastDoc = null
// This is the old way, we might want to deprecate it or make it call onEnterClass
// For now, if we are NOT using enter/exit, keep behavior.
// But Compiler.kt will be updated to use enter/exit.
if (classStack.isEmpty()) {
val attach = node.copy(doc = node.doc ?: lastDoc)
currentScript?.declarations?.add(attach)
lastDoc = null
}
}
override fun onEnumDecl(node: MiniEnumDecl) {