From 523b9d338b6d6659ab147aa0bd1bcd1df6b6f54e Mon Sep 17 00:00:00 2001 From: sergeych Date: Tue, 3 Feb 2026 02:07:29 +0300 Subject: [PATCH] Update compile-time resolution and tests --- AGENTS.md | 6 + docs/scopes_and_closures.md | 2 + .../kotlin/net/sergeych/lyng/CallSignature.kt | 24 + .../kotlin/net/sergeych/lyng/ClosureScope.kt | 21 +- .../kotlin/net/sergeych/lyng/CodeContext.kt | 11 +- .../kotlin/net/sergeych/lyng/Compiler.kt | 794 +++++++- .../net/sergeych/lyng/ExtensionNaming.kt | 34 + .../lyng/ExtensionPropertyDeclStatement.kt | 10 + .../net/sergeych/lyng/InlineBlockStatement.kt | 37 + .../kotlin/net/sergeych/lyng/Scope.kt | 57 +- .../kotlin/net/sergeych/lyng/Script.kt | 5 +- .../net/sergeych/lyng/VarDeclStatement.kt | 5 +- .../lyng/bytecode/BytecodeCompiler.kt | 1687 ++++++++++++++--- .../lyng/bytecode/BytecodeStatement.kt | 20 +- .../net/sergeych/lyng/bytecode/CmdBuilder.kt | 36 +- .../sergeych/lyng/bytecode/CmdDisassembler.kt | 37 +- .../net/sergeych/lyng/bytecode/CmdRuntime.kt | 134 +- .../net/sergeych/lyng/bytecode/Opcode.kt | 14 +- .../lyng/miniast/DocRegistrationHelpers.kt | 3 +- .../kotlin/net/sergeych/lyng/obj/Obj.kt | 41 +- .../kotlin/net/sergeych/lyng/obj/ObjClass.kt | 268 ++- .../sergeych/lyng/obj/ObjExtensionCallable.kt | 68 + .../kotlin/net/sergeych/lyng/obj/ObjFlow.kt | 9 +- .../net/sergeych/lyng/obj/ObjInstance.kt | 8 + .../net/sergeych/lyng/obj/ObjIterable.kt | 8 +- .../kotlin/net/sergeych/lyng/obj/ObjRecord.kt | 6 +- .../kotlin/net/sergeych/lyng/obj/ObjRef.kt | 463 ++--- .../kotlin/net/sergeych/lyng/obj/ObjRegex.kt | 17 +- .../kotlin/net/sergeych/lyng/obj/ObjString.kt | 34 +- .../kotlin/CompileTimeResolutionSpecTest.kt | 4 +- .../kotlin/ParallelLocalScopeTest.kt | 5 +- .../kotlin/ScopeCycleRegressionTest.kt | 4 +- lynglib/src/commonTest/kotlin/ScriptTest.kt | 386 ++-- .../kotlin/ScriptTest_OptionalAssign.kt | 4 +- .../lyng/miniast/ParamTypeInferenceTest.kt | 9 +- .../src/jvmTest/kotlin/MethodIdDebugTest.kt | 47 + .../src/jvmTest/kotlin/ScriptSubsetJvmTest.kt | 3 +- .../kotlin/ScriptSubsetJvmTest_Additions3.kt | 16 +- .../kotlin/ScriptSubsetJvmTest_Additions4.kt | 9 +- .../kotlin/ScriptSubsetJvmTest_Additions5.kt | 2 +- .../kotlin/ScriptSubsetJvmTest_additions.kt | 6 +- .../src/jvmTest/kotlin/StdlibWrapperTest.kt | 13 + lynglib/stdlib/lyng/root.lyng | 60 +- notes/ai_state.md | 52 + notes/compile_time_name_resolution_spec.md | 5 +- notes/new_lyng_type_system_spec.md | 254 +++ 46 files changed, 3771 insertions(+), 967 deletions(-) create mode 100644 lynglib/src/commonMain/kotlin/net/sergeych/lyng/CallSignature.kt create mode 100644 lynglib/src/commonMain/kotlin/net/sergeych/lyng/ExtensionNaming.kt create mode 100644 lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineBlockStatement.kt create mode 100644 lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjExtensionCallable.kt create mode 100644 lynglib/src/jvmTest/kotlin/MethodIdDebugTest.kt create mode 100644 lynglib/src/jvmTest/kotlin/StdlibWrapperTest.kt create mode 100644 notes/ai_state.md create mode 100644 notes/new_lyng_type_system_spec.md diff --git a/AGENTS.md b/AGENTS.md index fe03481..eb657db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,3 +6,9 @@ - If you need a wrapper for delegated properties, check for `getValue` explicitly and return a concrete `Statement` object when missing; avoid `onNotFoundResult` lambdas. - If wasmJs browser tests hang, first run `:lynglib:wasmJsNodeTest` and look for wasm compilation errors; hangs usually mean module instantiation failed. - Do not increase test timeouts to mask wasm generation errors; fix the invalid IR instead. + +## Type inference notes (notes/type_system_spec.md) +- Nullability is Kotlin-style: `T` non-null, `T?` nullable, `!!` asserts non-null. +- `void` is a singleton of class `Void` (syntax sugar for return type). +- Object member access requires explicit cast; remove `inspect` from Object and use `toInspectString()` instead. +- Do not reintroduce bytecode fallback opcodes (e.g., `GET_NAME`, `EVAL_*`, `CALL_FALLBACK`) or runtime name-resolution fallbacks; all symbol resolution must stay compile-time only. diff --git a/docs/scopes_and_closures.md b/docs/scopes_and_closures.md index 5e8b0f2..173ba1a 100644 --- a/docs/scopes_and_closures.md +++ b/docs/scopes_and_closures.md @@ -1,5 +1,7 @@ # Scopes and Closures: resolution and safety +Attention to AI: name lookup is ibsolete and must not be used with bytecode compiler + This page documents how name resolution works with `ClosureScope`, how to avoid recursion pitfalls, and how to safely capture and execute callbacks that need access to outer locals. ## Why this matters diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CallSignature.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CallSignature.kt new file mode 100644 index 0000000..6f0fb06 --- /dev/null +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CallSignature.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Sergey S. Chernov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sergeych.lyng + +/** + * Compile-time call metadata for known functions. Used to select lambda receiver semantics. + */ +data class CallSignature( + val tailBlockReceiverType: String? = null +) diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ClosureScope.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ClosureScope.kt index d1ef187..e2b8658 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ClosureScope.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ClosureScope.kt @@ -26,12 +26,25 @@ import net.sergeych.lyng.obj.ObjRecord * Inherits [Scope.args] and [Scope.thisObj] from [callScope] and adds lookup for symbols * from [closureScope] with proper precedence */ -class ClosureScope(val callScope: Scope, val closureScope: Scope) : +class ClosureScope( + val callScope: Scope, + val closureScope: Scope, + private val preferredThisType: String? = null +) : // Important: use closureScope.thisObj so unqualified members (e.g., fields) resolve to the instance // we captured, not to the caller's `this` (e.g., FlowBuilder). Scope(callScope, callScope.args, thisObj = closureScope.thisObj) { init { + val desired = preferredThisType?.let { typeName -> + callScope.thisVariants.firstOrNull { it.objClass.className == typeName } + } + val primaryThis = closureScope.thisObj + val merged = ArrayList(callScope.thisVariants.size + closureScope.thisVariants.size + 1) + desired?.let { merged.add(it) } + merged.addAll(callScope.thisVariants) + merged.addAll(closureScope.thisVariants) + setThisVariants(primaryThis, merged) // Preserve the lexical class context of the closure by default. This ensures that lambdas // created inside a class method keep access to that class's private/protected members even // when executed from within another object's method (e.g., Mutex.withLock), which may set @@ -72,14 +85,14 @@ class ClosureScope(val callScope: Scope, val closureScope: Scope) : } class ApplyScope(val callScope: Scope, val applied: Scope) : - Scope(callScope.parent?.parent ?: callScope.parent ?: callScope, thisObj = applied.thisObj) { + Scope(callScope, thisObj = applied.thisObj) { override fun get(name: String): ObjRecord? { return applied.get(name) ?: super.get(name) } - override fun applyClosure(closure: Scope): Scope { - return this + override fun applyClosure(closure: Scope, preferredThisType: String?): Scope { + return ClosureScope(this, closure, preferredThisType) } } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CodeContext.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CodeContext.kt index f8fc618..bc9773d 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CodeContext.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/CodeContext.kt @@ -19,10 +19,19 @@ package net.sergeych.lyng sealed class CodeContext { class Module(@Suppress("unused") val packageName: String?): CodeContext() - class Function(val name: String, val implicitThisMembers: Boolean = false): CodeContext() + class Function( + val name: String, + val implicitThisMembers: Boolean = false, + val implicitThisTypeName: String? = null + ): CodeContext() class ClassBody(val name: String, val isExtern: Boolean = false): CodeContext() { val pendingInitializations = mutableMapOf() val declaredMembers = mutableSetOf() + val memberOverrides = mutableMapOf() + val memberFieldIds = mutableMapOf() + val memberMethodIds = mutableMapOf() + var nextFieldId: Int = 0 + var nextMethodId: Int = 0 var slotPlanId: Int? = null } } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt index 5c0d5db..91a3148 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt @@ -19,6 +19,9 @@ package net.sergeych.lyng import net.sergeych.lyng.Compiler.Companion.compile import net.sergeych.lyng.bytecode.BytecodeStatement +import net.sergeych.lyng.bytecode.CmdListLiteral +import net.sergeych.lyng.bytecode.CmdMakeRange +import net.sergeych.lyng.bytecode.CmdRangeIntBounds import net.sergeych.lyng.miniast.* import net.sergeych.lyng.obj.* import net.sergeych.lyng.pacman.ImportManager @@ -109,6 +112,11 @@ class Compiler( if (!record.visibility.isPublic) continue declareSlotNameIn(plan, name, record.isMutable, record.type == ObjRecord.Type.Delegated) } + for ((name, slotIndex) in current.slotNameToIndexSnapshot()) { + val record = current.getSlotRecord(slotIndex) + if (!record.visibility.isPublic) continue + declareSlotNameIn(plan, name, record.isMutable, record.type == ObjRecord.Type.Delegated) + } if (!includeParents) return current = current.parent } @@ -142,6 +150,8 @@ class Compiler( val actual = cc.nextNonWhitespace() if (actual.type == Token.Type.ID) { extensionNames.add(actual.value) + registerExtensionName(nameToken.value, actual.value) + declareSlotNameIn(plan, extensionCallableName(nameToken.value, actual.value), isMutable = false, isDelegated = false) } continue } @@ -156,6 +166,11 @@ class Compiler( val actual = cc.nextNonWhitespace() if (actual.type == Token.Type.ID) { extensionNames.add(actual.value) + registerExtensionName(nameToken.value, actual.value) + declareSlotNameIn(plan, extensionPropertyGetterName(nameToken.value, actual.value), isMutable = false, isDelegated = false) + if (t.value == "var") { + declareSlotNameIn(plan, extensionPropertySetterName(nameToken.value, actual.value), isMutable = false, isDelegated = false) + } } continue } @@ -184,7 +199,7 @@ class Compiler( } } - private fun predeclareClassMembers(target: MutableSet) { + private fun predeclareClassMembers(target: MutableSet, overrides: MutableMap) { val saved = cc.savePos() var depth = 0 val modifiers = setOf( @@ -205,7 +220,9 @@ class Compiler( Token.Type.LBRACE -> depth++ Token.Type.RBRACE -> if (depth == 0) break else depth-- Token.Type.ID -> if (depth == 0) { + var sawOverride = false while (t.type == Token.Type.ID && t.value in modifiers) { + if (t.value == "override") sawOverride = true t = nextNonWs() } when (t.value) { @@ -215,6 +232,7 @@ class Compiler( val afterName = cc.peekNextNonWhitespace() if (afterName.type != Token.Type.DOT) { target.add(nameToken.value) + overrides[nameToken.value] = sawOverride } } } @@ -241,6 +259,57 @@ class Compiler( } } + private fun resolveCompileClassInfo(name: String): CompileClassInfo? { + compileClassInfos[name]?.let { return it } + val scopeRec = seedScope?.get(name) ?: importManager.rootScope.get(name) + val cls = scopeRec?.value as? ObjClass ?: return null + val fieldIds = cls.instanceFieldIdMap() + val methodIds = cls.instanceMethodIdMap(includeAbstract = true) + val nextFieldId = (fieldIds.values.maxOrNull() ?: -1) + 1 + val nextMethodId = (methodIds.values.maxOrNull() ?: -1) + 1 + return CompileClassInfo(name, fieldIds, methodIds, nextFieldId, nextMethodId) + } + + private data class BaseMemberIds( + val fieldIds: Map, + val methodIds: Map, + val fieldConflicts: Set, + val methodConflicts: Set, + val nextFieldId: Int, + val nextMethodId: Int + ) + + private fun collectBaseMemberIds(baseNames: List): BaseMemberIds { + val allBaseNames = if (baseNames.contains("Object")) baseNames else baseNames + "Object" + val fieldIds = mutableMapOf() + val methodIds = mutableMapOf() + val fieldConflicts = mutableSetOf() + val methodConflicts = mutableSetOf() + var maxFieldId = -1 + var maxMethodId = -1 + for (base in allBaseNames) { + val info = resolveCompileClassInfo(base) ?: continue + for ((name, id) in info.fieldIds) { + val prev = fieldIds.putIfAbsent(name, id) + if (prev != null && prev != id) fieldConflicts.add(name) + if (id > maxFieldId) maxFieldId = id + } + for ((name, id) in info.methodIds) { + val prev = methodIds.putIfAbsent(name, id) + if (prev != null && prev != id) methodConflicts.add(name) + if (id > maxMethodId) maxMethodId = id + } + } + return BaseMemberIds( + fieldIds = fieldIds, + methodIds = methodIds, + fieldConflicts = fieldConflicts, + methodConflicts = methodConflicts, + nextFieldId = maxFieldId + 1, + nextMethodId = maxMethodId + 1 + ) + } + private fun buildParamSlotPlan(names: List): SlotPlan { val map = mutableMapOf() var idx = 0 @@ -274,6 +343,62 @@ class Compiler( return result } + private fun callSignatureForName(name: String): CallSignature? { + seedScope?.getLocalRecordDirect(name)?.callSignature?.let { return it } + return seedScope?.get(name)?.callSignature + ?: importManager.rootScope.getLocalRecordDirect(name)?.callSignature + } + + internal data class MemberIds(val fieldId: Int?, val methodId: Int?) + + private fun resolveMemberIds(name: String, pos: Pos, qualifier: String? = null): MemberIds { + val ctx = if (qualifier == null) { + codeContexts.asReversed().firstOrNull { it is CodeContext.ClassBody } as? CodeContext.ClassBody + } else null + if (ctx != null) { + val fieldId = ctx.memberFieldIds[name] + val methodId = ctx.memberMethodIds[name] + if (fieldId == null && methodId == null) { + if (allowUnresolvedRefs) return MemberIds(null, null) + throw ScriptError(pos, "unknown member $name") + } + return MemberIds(fieldId, methodId) + } + if (qualifier != null) { + val info = resolveCompileClassInfo(qualifier) + ?: if (allowUnresolvedRefs) return MemberIds(null, null) else throw ScriptError(pos, "unknown type $qualifier") + val fieldId = info.fieldIds[name] + val methodId = info.methodIds[name] + if (fieldId == null && methodId == null) { + if (allowUnresolvedRefs) return MemberIds(null, null) + throw ScriptError(pos, "unknown member $name on $qualifier") + } + return MemberIds(fieldId, methodId) + } + if (allowUnresolvedRefs) return MemberIds(null, null) + throw ScriptError(pos, "member $name is not available without class context") + } + + private fun tailBlockReceiverType(left: ObjRef): String? { + val name = when (left) { + is LocalVarRef -> left.name + is LocalSlotRef -> left.name + is ImplicitThisMemberRef -> left.name + else -> null + } + if (name == null) return null + val signature = callSignatureForName(name) + return signature?.tailBlockReceiverType ?: if (name == "flow") "FlowBuilder" else null + } + + private fun currentImplicitThisTypeName(): String? { + for (ctx in codeContexts.asReversed()) { + val fn = ctx as? CodeContext.Function ?: continue + if (fn.implicitThisTypeName != null) return fn.implicitThisTypeName + } + return null + } + private fun lookupSlotLocation(name: String, includeModule: Boolean = true): SlotLocation? { for (i in slotPlanStack.indices.reversed()) { if (!includeModule && i == 0) continue @@ -290,10 +415,6 @@ class Compiler( val value = ObjString(packageName ?: "unknown").asReadonly return ConstRef(value) } - if (name == "$~") { - resolutionSink?.reference(name, pos) - return LocalVarRef(name, pos) - } if (name == "this") { resolutionSink?.reference(name, pos) return LocalVarRef(name, pos) @@ -306,7 +427,8 @@ class Compiler( classCtx.declaredMembers.contains(name) ) { resolutionSink?.referenceMember(name, pos) - return ImplicitThisMemberRef(name, pos) + val ids = resolveMemberIds(name, pos, null) + return ImplicitThisMemberRef(name, pos, ids.fieldId, ids.methodId, currentImplicitThisTypeName()) } captureLocalRef(name, slotLoc, pos)?.let { ref -> resolutionSink?.reference(name, pos) @@ -343,7 +465,8 @@ class Compiler( val classCtx = codeContexts.lastOrNull { it is CodeContext.ClassBody } as? CodeContext.ClassBody if (classCtx != null && classCtx.declaredMembers.contains(name)) { resolutionSink?.referenceMember(name, pos) - return ImplicitThisMemberRef(name, pos) + val ids = resolveMemberIds(name, pos, null) + return ImplicitThisMemberRef(name, pos, ids.fieldId, ids.methodId, currentImplicitThisTypeName()) } val modulePlan = moduleSlotPlan() val moduleEntry = modulePlan?.slots?.get(name) @@ -395,8 +518,10 @@ class Compiler( (ctx as? CodeContext.Function)?.implicitThisMembers == true } if (implicitThis) { - resolutionSink?.referenceMember(name, pos) - return ImplicitThisMemberRef(name, pos) + val implicitType = currentImplicitThisTypeName() + resolutionSink?.referenceMember(name, pos, implicitType) + val ids = resolveImplicitThisMemberIds(name, pos, implicitType) + return ImplicitThisMemberRef(name, pos, ids.fieldId, ids.methodId, implicitType) } val classContext = codeContexts.any { ctx -> ctx is CodeContext.ClassBody } if (classContext && extensionNames.contains(name)) { @@ -477,6 +602,13 @@ class Compiler( } } + private fun shouldSeedDefaultStdlib(): Boolean { + if (seedScope != null) return false + if (importManager !== Script.defaultImportManager) return false + val sourceName = cc.tokens.firstOrNull()?.pos?.source?.fileName + return sourceName != "lyng.stdlib" + } + private var anonCounter = 0 private fun generateAnonName(pos: Pos): String { return "${"$"}${"Anon"}_${pos.line+1}_${pos.column}_${++anonCounter}" @@ -527,6 +659,17 @@ class Compiler( private val initStack = mutableListOf>() + private data class CompileClassInfo( + val name: String, + val fieldIds: Map, + val methodIds: Map, + val nextFieldId: Int, + val nextMethodId: Int + ) + + private val compileClassInfos = mutableMapOf() + private val compileClassStubs = mutableMapOf() + val currentInitScope: MutableList get() = initStack.lastOrNull() ?: cc.syntaxError("no initialization scope exists here") @@ -571,8 +714,14 @@ class Compiler( if (needsSlotPlan) { slotPlanStack.add(SlotPlan(mutableMapOf(), 0, nextScopeId++)) declareSlotNameIn(slotPlanStack.last(), "__PACKAGE__", isMutable = false, isDelegated = false) - seedScope?.let { seedSlotPlanFromScope(it) } + declareSlotNameIn(slotPlanStack.last(), "$~", isMutable = true, isDelegated = false) + seedScope?.let { seedSlotPlanFromScope(it, includeParents = true) } seedSlotPlanFromScope(importManager.rootScope) + if (shouldSeedDefaultStdlib()) { + val stdlib = importManager.prepareImport(start, "lyng.stdlib", null) + seedResolutionFromScope(stdlib, start) + seedSlotPlanFromScope(stdlib) + } predeclareTopLevelSymbols() } return try { @@ -678,7 +827,26 @@ class Compiler( } while (true) val modulePlan = if (needsSlotPlan) slotPlanIndices(slotPlanStack.last()) else emptyMap() - Script(start, statements, modulePlan) + val wrapScriptBytecode = useBytecodeStatements && + statements.isNotEmpty() && + codeContexts.lastOrNull() is CodeContext.Module && + resolutionScriptDepth == 1 && + statements.none { containsUnsupportedForBytecode(it) } + val finalStatements = if (wrapScriptBytecode) { + val unwrapped = statements.map { unwrapBytecodeDeep(it) } + val block = InlineBlockStatement(unwrapped, start) + listOf( + BytecodeStatement.wrap( + block, + "