Fix bytecode name lookup; unignore more stdlib tests

This commit is contained in:
Sergey Chernov 2026-01-29 09:57:29 +03:00
parent e346e7e56e
commit d8e18e4a0c
7 changed files with 236 additions and 58 deletions

View File

@ -31,6 +31,7 @@ import net.sergeych.lyng.WhenInCondition
import net.sergeych.lyng.WhenIsCondition
import net.sergeych.lyng.WhenStatement
import net.sergeych.lyng.obj.*
import java.util.IdentityHashMap
class BytecodeCompiler(
private val allowLocalSlots: Boolean = true,
@ -62,7 +63,8 @@ class BytecodeCompiler(
private val slotTypes = mutableMapOf<Int, SlotType>()
private val intLoopVarNames = LinkedHashSet<String>()
private val loopStack = ArrayDeque<LoopContext>()
private val virtualScopeDepths = LinkedHashSet<Int>()
private val effectiveScopeDepthByRef = IdentityHashMap<LocalSlotRef, Int>()
private val effectiveLocalDepthByKey = LinkedHashMap<ScopeSlotKey, Int>()
private data class LoopContext(
val label: String?,
@ -183,6 +185,9 @@ class BytecodeCompiler(
if (!allowLocalSlots) return null
if (ref.isDelegated) return null
if (ref.name.isEmpty()) return null
if (refDepth(ref) > 0) {
return compileNameLookup(ref.name)
}
val mapped = resolveSlot(ref) ?: return compileNameLookup(ref.name)
var resolved = slotTypes[mapped] ?: SlotType.UNKNOWN
if (resolved == SlotType.UNKNOWN && intLoopVarNames.contains(ref.name)) {
@ -198,7 +203,21 @@ class BytecodeCompiler(
}
CompiledValue(mapped, resolved)
}
is LocalVarRef -> compileNameLookup(ref.name)
is LocalVarRef -> {
if (allowLocalSlots) {
loopSlotOverrides[ref.name]?.let { slot ->
val resolved = slotTypes[slot] ?: SlotType.UNKNOWN
return CompiledValue(slot, resolved)
}
val localIndex = localSlotIndexByName[ref.name]
if (localIndex != null) {
val slot = scopeSlotCount + localIndex
val resolved = slotTypes[slot] ?: SlotType.UNKNOWN
return CompiledValue(slot, resolved)
}
}
compileNameLookup(ref.name)
}
is ValueFnRef -> {
val constId = builder.addConst(BytecodeConst.ValueFn(ref.valueFn()))
val slot = allocSlot()
@ -946,9 +965,19 @@ class BytecodeCompiler(
if (localTarget != null) {
if (!allowLocalSlots) return compileEvalRef(ref)
if (localTarget.isDelegated) return compileEvalRef(ref)
if (!localTarget.isMutable) return compileEvalRef(ref)
val slot = resolveSlot(localTarget) ?: return null
val targetType = slotTypes[slot] ?: SlotType.OBJ
if (!localTarget.isMutable) {
if (targetType != SlotType.OBJ && targetType != SlotType.UNKNOWN) return compileEvalRef(ref)
val rhs = compileRef(ref.value) ?: return compileEvalRef(ref)
val rhsObj = ensureObjSlot(rhs)
val nameId = builder.addConst(BytecodeConst.StringVal(localTarget.name))
if (nameId > 0xFFFF) return compileEvalRef(ref)
val dst = allocSlot()
builder.emit(Opcode.ASSIGN_OP_OBJ, ref.op.ordinal, slot, rhsObj.slot, dst, nameId)
updateSlotType(dst, SlotType.OBJ)
return CompiledValue(dst, SlotType.OBJ)
}
var rhs = compileRef(ref.value) ?: return compileEvalRef(ref)
if (targetType == SlotType.OBJ && rhs.type != SlotType.OBJ) {
rhs = ensureObjSlot(rhs)
@ -2875,11 +2904,12 @@ class BytecodeCompiler(
intLoopVarNames.clear()
addrSlotByScopeSlot.clear()
loopStack.clear()
virtualScopeDepths.clear()
effectiveScopeDepthByRef.clear()
effectiveLocalDepthByKey.clear()
if (allowLocalSlots) {
collectLoopVarNames(stmt)
}
collectVirtualScopeDepths(stmt, 0)
collectEffectiveDepths(stmt, 0, ArrayDeque())
collectScopeSlots(stmt)
if (allowLocalSlots) {
collectLoopSlotPlans(stmt, 0)
@ -2917,7 +2947,8 @@ class BytecodeCompiler(
}
names.add(info.name)
mutables[index] = info.isMutable
depths[index] = effectiveLocalDepth(info.depth)
val effectiveDepth = effectiveLocalDepthByKey[key] ?: info.depth
depths[index] = effectiveDepth
index += 1
}
localSlotNames = names.toTypedArray()
@ -3155,7 +3186,7 @@ class BytecodeCompiler(
when (ref) {
is LocalSlotRef -> {
val localKey = ScopeSlotKey(refScopeDepth(ref), refSlot(ref))
val shouldLocalize = declaredLocalKeys.contains(localKey) ||
val shouldLocalize = (refDepth(ref) == 0) ||
intLoopVarNames.contains(ref.name)
if (allowLocalSlots && !ref.isDelegated && shouldLocalize) {
if (!localSlotInfoMap.containsKey(localKey)) {
@ -3181,7 +3212,7 @@ class BytecodeCompiler(
val target = assignTarget(ref)
if (target != null) {
val localKey = ScopeSlotKey(refScopeDepth(target), refSlot(target))
val shouldLocalize = declaredLocalKeys.contains(localKey) ||
val shouldLocalize = (refDepth(target) == 0) ||
intLoopVarNames.contains(target.name)
if (allowLocalSlots && !target.isDelegated && shouldLocalize) {
if (!localSlotInfoMap.containsKey(localKey)) {
@ -3243,75 +3274,176 @@ class BytecodeCompiler(
}
}
private fun collectVirtualScopeDepths(stmt: Statement, scopeDepth: Int) {
private fun collectEffectiveDepths(
stmt: Statement,
scopeDepth: Int,
virtualDepths: ArrayDeque<Int>,
) {
if (stmt is BytecodeStatement) {
collectVirtualScopeDepths(stmt.original, scopeDepth)
collectEffectiveDepths(stmt.original, scopeDepth, virtualDepths)
return
}
when (stmt) {
is net.sergeych.lyng.ForInStatement -> {
collectVirtualScopeDepths(stmt.source, scopeDepth)
collectEffectiveDepths(stmt.source, scopeDepth, virtualDepths)
val loopDepth = scopeDepth + 1
virtualScopeDepths.add(loopDepth)
val bodyTarget = if (stmt.body is BytecodeStatement) stmt.body.original else stmt.body
if (bodyTarget is BlockStatement) {
// Loop bodies are inlined in bytecode, so their block scope is virtual.
virtualScopeDepths.add(loopDepth + 1)
virtualDepths.addLast(loopDepth)
if (allowLocalSlots) {
for ((_, slotIndex) in stmt.loopSlotPlan) {
val key = ScopeSlotKey(loopDepth, slotIndex)
if (!effectiveLocalDepthByKey.containsKey(key)) {
effectiveLocalDepthByKey[key] = calcEffectiveLocalDepth(loopDepth, virtualDepths)
}
collectVirtualScopeDepths(stmt.body, loopDepth)
stmt.elseStatement?.let { collectVirtualScopeDepths(it, loopDepth) }
}
}
val bodyTarget = if (stmt.body is BytecodeStatement) stmt.body.original else stmt.body
val bodyIsBlock = bodyTarget is BlockStatement
if (bodyIsBlock) {
// Loop bodies are inlined in bytecode, so their block scope is virtual.
virtualDepths.addLast(loopDepth + 1)
}
collectEffectiveDepths(stmt.body, loopDepth, virtualDepths)
if (bodyIsBlock) {
virtualDepths.removeLast()
}
stmt.elseStatement?.let { collectEffectiveDepths(it, loopDepth, virtualDepths) }
virtualDepths.removeLast()
}
is net.sergeych.lyng.WhileStatement -> {
collectVirtualScopeDepths(stmt.condition, scopeDepth)
collectEffectiveDepths(stmt.condition, scopeDepth, virtualDepths)
val loopDepth = scopeDepth + 1
virtualScopeDepths.add(loopDepth)
collectVirtualScopeDepths(stmt.body, loopDepth)
stmt.elseStatement?.let { collectVirtualScopeDepths(it, loopDepth) }
virtualDepths.addLast(loopDepth)
if (allowLocalSlots) {
for ((_, slotIndex) in stmt.loopSlotPlan) {
val key = ScopeSlotKey(loopDepth, slotIndex)
if (!effectiveLocalDepthByKey.containsKey(key)) {
effectiveLocalDepthByKey[key] = calcEffectiveLocalDepth(loopDepth, virtualDepths)
}
}
}
collectEffectiveDepths(stmt.body, loopDepth, virtualDepths)
stmt.elseStatement?.let { collectEffectiveDepths(it, loopDepth, virtualDepths) }
virtualDepths.removeLast()
}
is net.sergeych.lyng.DoWhileStatement -> {
val loopDepth = scopeDepth + 1
virtualScopeDepths.add(loopDepth)
collectVirtualScopeDepths(stmt.body, loopDepth)
collectVirtualScopeDepths(stmt.condition, loopDepth)
stmt.elseStatement?.let { collectVirtualScopeDepths(it, loopDepth) }
virtualDepths.addLast(loopDepth)
if (allowLocalSlots) {
for ((_, slotIndex) in stmt.loopSlotPlan) {
val key = ScopeSlotKey(loopDepth, slotIndex)
if (!effectiveLocalDepthByKey.containsKey(key)) {
effectiveLocalDepthByKey[key] = calcEffectiveLocalDepth(loopDepth, virtualDepths)
}
}
}
collectEffectiveDepths(stmt.body, loopDepth, virtualDepths)
collectEffectiveDepths(stmt.condition, loopDepth, virtualDepths)
stmt.elseStatement?.let { collectEffectiveDepths(it, loopDepth, virtualDepths) }
virtualDepths.removeLast()
}
is BlockStatement -> {
val nextDepth = scopeDepth + 1
for (child in stmt.statements()) {
collectVirtualScopeDepths(child, nextDepth)
collectEffectiveDepths(child, nextDepth, virtualDepths)
}
}
is IfStatement -> {
collectVirtualScopeDepths(stmt.condition, scopeDepth)
collectVirtualScopeDepths(stmt.ifBody, scopeDepth)
stmt.elseBody?.let { collectVirtualScopeDepths(it, scopeDepth) }
collectEffectiveDepths(stmt.condition, scopeDepth, virtualDepths)
collectEffectiveDepths(stmt.ifBody, scopeDepth, virtualDepths)
stmt.elseBody?.let { collectEffectiveDepths(it, scopeDepth, virtualDepths) }
}
is VarDeclStatement -> {
stmt.initializer?.let { collectVirtualScopeDepths(it, scopeDepth) }
val slotIndex = stmt.slotIndex
val slotDepth = stmt.slotDepth
if (allowLocalSlots && slotIndex != null && slotDepth != null) {
val key = ScopeSlotKey(slotDepth, slotIndex)
if (!effectiveLocalDepthByKey.containsKey(key)) {
effectiveLocalDepthByKey[key] = calcEffectiveLocalDepth(slotDepth, virtualDepths)
}
is ExpressionStatement -> {
// no-op
}
stmt.initializer?.let { collectEffectiveDepths(it, scopeDepth, virtualDepths) }
}
is ExpressionStatement -> collectEffectiveDepthsRef(stmt.ref, virtualDepths)
is net.sergeych.lyng.BreakStatement -> {
stmt.resultExpr?.let { collectVirtualScopeDepths(it, scopeDepth) }
stmt.resultExpr?.let { collectEffectiveDepths(it, scopeDepth, virtualDepths) }
}
is net.sergeych.lyng.ReturnStatement -> {
stmt.resultExpr?.let { collectVirtualScopeDepths(it, scopeDepth) }
stmt.resultExpr?.let { collectEffectiveDepths(it, scopeDepth, virtualDepths) }
}
is net.sergeych.lyng.ThrowStatement -> {
collectVirtualScopeDepths(stmt.throwExpr, scopeDepth)
collectEffectiveDepths(stmt.throwExpr, scopeDepth, virtualDepths)
}
else -> {}
}
}
private fun effectiveScopeDepth(ref: LocalSlotRef): Int {
private fun collectEffectiveDepthsRef(ref: ObjRef, virtualDepths: ArrayDeque<Int>) {
when (ref) {
is LocalSlotRef -> {
if (!effectiveScopeDepthByRef.containsKey(ref)) {
effectiveScopeDepthByRef[ref] = calcEffectiveScopeDepth(ref, virtualDepths)
}
}
is BinaryOpRef -> {
collectEffectiveDepthsRef(binaryLeft(ref), virtualDepths)
collectEffectiveDepthsRef(binaryRight(ref), virtualDepths)
}
is UnaryOpRef -> collectEffectiveDepthsRef(unaryOperand(ref), virtualDepths)
is AssignRef -> {
collectEffectiveDepthsRef(assignValue(ref), virtualDepths)
assignTarget(ref)?.let { collectEffectiveDepthsRef(it, virtualDepths) }
}
is AssignOpRef -> {
collectEffectiveDepthsRef(ref.target, virtualDepths)
collectEffectiveDepthsRef(ref.value, virtualDepths)
}
is AssignIfNullRef -> {
collectEffectiveDepthsRef(ref.target, virtualDepths)
collectEffectiveDepthsRef(ref.value, virtualDepths)
}
is IncDecRef -> collectEffectiveDepthsRef(ref.target, virtualDepths)
is ConditionalRef -> {
collectEffectiveDepthsRef(ref.condition, virtualDepths)
collectEffectiveDepthsRef(ref.ifTrue, virtualDepths)
collectEffectiveDepthsRef(ref.ifFalse, virtualDepths)
}
is ElvisRef -> {
collectEffectiveDepthsRef(ref.left, virtualDepths)
collectEffectiveDepthsRef(ref.right, virtualDepths)
}
is FieldRef -> collectEffectiveDepthsRef(ref.target, virtualDepths)
is IndexRef -> {
collectEffectiveDepthsRef(ref.targetRef, virtualDepths)
collectEffectiveDepthsRef(ref.indexRef, virtualDepths)
}
is CallRef -> {
collectEffectiveDepthsRef(ref.target, virtualDepths)
collectEffectiveDepthsArgs(ref.args, virtualDepths)
}
is MethodCallRef -> {
collectEffectiveDepthsRef(ref.receiver, virtualDepths)
collectEffectiveDepthsArgs(ref.args, virtualDepths)
}
else -> {}
}
}
private fun collectEffectiveDepthsArgs(args: List<ParsedArgument>, virtualDepths: ArrayDeque<Int>) {
for (arg in args) {
val stmt = arg.value
if (stmt is ExpressionStatement) {
collectEffectiveDepthsRef(stmt.ref, virtualDepths)
}
}
}
private fun calcEffectiveScopeDepth(ref: LocalSlotRef, virtualDepths: ArrayDeque<Int>): Int {
val baseDepth = refDepth(ref)
if (baseDepth == 0 || virtualScopeDepths.isEmpty()) return baseDepth
if (baseDepth == 0 || virtualDepths.isEmpty()) return baseDepth
val targetDepth = refScopeDepth(ref)
val currentDepth = targetDepth + baseDepth
var virtualCount = 0
for (depth in virtualScopeDepths) {
for (depth in virtualDepths) {
if (depth > targetDepth && depth <= currentDepth) {
virtualCount += 1
}
@ -3319,6 +3451,21 @@ class BytecodeCompiler(
return baseDepth - virtualCount
}
private fun calcEffectiveLocalDepth(depth: Int, virtualDepths: ArrayDeque<Int>): Int {
if (depth == 0 || virtualDepths.isEmpty()) return depth
var virtualCount = 0
for (virtualDepth in virtualDepths) {
if (virtualDepth <= depth) {
virtualCount += 1
}
}
return depth - virtualCount
}
private fun effectiveScopeDepth(ref: LocalSlotRef): Int {
return effectiveScopeDepthByRef[ref] ?: refDepth(ref)
}
private fun extractRangeRef(source: Statement): RangeRef? {
val target = if (source is BytecodeStatement) source.original else source
val expr = target as? ExpressionStatement ?: return null
@ -3349,16 +3496,5 @@ class BytecodeCompiler(
return if (rangeLocalNames.contains(localRef.name)) localRef else null
}
private fun effectiveLocalDepth(depth: Int): Int {
if (depth == 0 || virtualScopeDepths.isEmpty()) return depth
var virtualCount = 0
for (virtualDepth in virtualScopeDepths) {
if (virtualDepth <= depth) {
virtualCount += 1
}
}
return depth - virtualCount
}
private data class ScopeSlotKey(val depth: Int, val slot: Int)
}

View File

@ -36,7 +36,7 @@ class BytecodeStatement private constructor(
override val pos: Pos = original.pos
override suspend fun execute(scope: Scope): Obj {
return CmdVm().execute(function, scope, emptyList())
return CmdVm().execute(function, scope, scope.args.list)
}
internal fun bytecodeFunction(): CmdFunction = function

View File

@ -160,6 +160,8 @@ class CmdBuilder {
Opcode.ADD_OBJ, Opcode.SUB_OBJ, Opcode.MUL_OBJ, Opcode.DIV_OBJ, Opcode.MOD_OBJ, Opcode.CONTAINS_OBJ,
Opcode.AND_BOOL, Opcode.OR_BOOL ->
listOf(OperandKind.SLOT, OperandKind.SLOT, OperandKind.SLOT)
Opcode.ASSIGN_OP_OBJ ->
listOf(OperandKind.ID, OperandKind.SLOT, OperandKind.SLOT, OperandKind.SLOT, OperandKind.CONST)
Opcode.INC_INT, Opcode.DEC_INT, Opcode.RET ->
listOf(OperandKind.SLOT)
Opcode.JMP ->
@ -364,6 +366,7 @@ class CmdBuilder {
Opcode.DIV_OBJ -> CmdDivObj(operands[0], operands[1], operands[2])
Opcode.MOD_OBJ -> CmdModObj(operands[0], operands[1], operands[2])
Opcode.CONTAINS_OBJ -> CmdContainsObj(operands[0], operands[1], operands[2])
Opcode.ASSIGN_OP_OBJ -> CmdAssignOpObj(operands[0], operands[1], operands[2], operands[3], operands[4])
Opcode.JMP -> CmdJmp(operands[0])
Opcode.JMP_IF_TRUE -> CmdJmpIfTrue(operands[0], operands[1])
Opcode.JMP_IF_FALSE -> CmdJmpIfFalse(operands[0], operands[1])

View File

@ -162,6 +162,7 @@ object CmdDisassembler {
is CmdDivObj -> Opcode.DIV_OBJ to intArrayOf(cmd.a, cmd.b, cmd.dst)
is CmdModObj -> Opcode.MOD_OBJ to intArrayOf(cmd.a, cmd.b, cmd.dst)
is CmdContainsObj -> Opcode.CONTAINS_OBJ to intArrayOf(cmd.target, cmd.value, cmd.dst)
is CmdAssignOpObj -> Opcode.ASSIGN_OP_OBJ to intArrayOf(cmd.opId, cmd.targetSlot, cmd.valueSlot, cmd.dst, cmd.nameId)
is CmdJmp -> Opcode.JMP to intArrayOf(cmd.target)
is CmdJmpIfTrue -> Opcode.JMP_IF_TRUE to intArrayOf(cmd.cond, cmd.target)
is CmdJmpIfFalse -> Opcode.JMP_IF_FALSE to intArrayOf(cmd.cond, cmd.target)
@ -252,6 +253,8 @@ object CmdDisassembler {
Opcode.ADD_OBJ, Opcode.SUB_OBJ, Opcode.MUL_OBJ, Opcode.DIV_OBJ, Opcode.MOD_OBJ, Opcode.CONTAINS_OBJ,
Opcode.AND_BOOL, Opcode.OR_BOOL ->
listOf(OperandKind.SLOT, OperandKind.SLOT, OperandKind.SLOT)
Opcode.ASSIGN_OP_OBJ ->
listOf(OperandKind.ID, OperandKind.SLOT, OperandKind.SLOT, OperandKind.SLOT, OperandKind.CONST)
Opcode.INC_INT, Opcode.DEC_INT, Opcode.RET, Opcode.ITER_PUSH ->
listOf(OperandKind.SLOT)
Opcode.JMP ->

View File

@ -17,6 +17,7 @@
package net.sergeych.lyng.bytecode
import net.sergeych.lyng.Arguments
import net.sergeych.lyng.ExecutionError
import net.sergeych.lyng.PerfFlags
import net.sergeych.lyng.PerfStats
import net.sergeych.lyng.Pos
@ -32,6 +33,9 @@ class CmdVm {
result = null
val frame = CmdFrame(this, fn, scope0, args)
val cmds = fn.cmds
if (fn.localSlotNames.isNotEmpty()) {
frame.syncScopeToFrame()
}
try {
while (result == null) {
val cmd = cmds[frame.ip]
@ -941,6 +945,34 @@ class CmdContainsObj(internal val target: Int, internal val value: Int, internal
}
}
class CmdAssignOpObj(
internal val opId: Int,
internal val targetSlot: Int,
internal val valueSlot: Int,
internal val dst: Int,
internal val nameId: Int,
) : Cmd() {
override suspend fun perform(frame: CmdFrame) {
val target = frame.slotToObj(targetSlot)
val value = frame.slotToObj(valueSlot)
val result = when (BinOp.values().getOrNull(opId)) {
BinOp.PLUS -> target.plusAssign(frame.scope, value)
BinOp.MINUS -> target.minusAssign(frame.scope, value)
BinOp.STAR -> target.mulAssign(frame.scope, value)
BinOp.SLASH -> target.divAssign(frame.scope, value)
BinOp.PERCENT -> target.modAssign(frame.scope, value)
else -> null
}
if (result == null) {
val name = (frame.fn.constants.getOrNull(nameId) as? BytecodeConst.StringVal)?.value
if (name != null) frame.scope.raiseIllegalAssignment("symbol is readonly: $name")
frame.scope.raiseIllegalAssignment("symbol is readonly")
}
frame.storeObjResult(dst, result)
return
}
}
class CmdJmp(internal val target: Int) : Cmd() {
override suspend fun perform(frame: CmdFrame) {
frame.ip = target
@ -1175,7 +1207,7 @@ class CmdCallSlot(
frame.fn.localSlotNames.getOrNull(localIndex)
}
val message = name?.let { "property '$it' is unset (not initialized)" }
?: "property is unset (not initialized)"
?: "property is unset (not initialized) in ${frame.fn.name} at slot $calleeSlot"
frame.scope.raiseUnset(message)
}
val args = frame.buildArguments(argBase, argCount)
@ -1238,7 +1270,14 @@ class CmdGetName(
}
val nameConst = frame.fn.constants.getOrNull(nameId) as? BytecodeConst.StringVal
?: error("GET_NAME expects StringVal at $nameId")
val result = frame.scope.get(nameConst.value)?.value ?: ObjUnset
val name = nameConst.value
val result = frame.scope.get(name)?.value ?: run {
try {
frame.scope.thisObj.readField(frame.scope, name).value
} catch (e: ExecutionError) {
if ((e.message ?: "").contains("no such field: $name")) ObjUnset else throw e
}
}
frame.storeObjResult(dst, result)
return
}

View File

@ -107,6 +107,7 @@ enum class Opcode(val code: Int) {
DIV_OBJ(0x7A),
MOD_OBJ(0x7B),
CONTAINS_OBJ(0x7C),
ASSIGN_OP_OBJ(0x7D),
JMP(0x80),
JMP_IF_TRUE(0x81),

View File

@ -33,7 +33,6 @@ class StdlibTest {
}
@Test
@Ignore("TODO(bytecode-only): range first/last mismatch")
fun testFirstLast() = runTest {
eval("""
assertEquals(1, (1..8).first )
@ -42,7 +41,6 @@ class StdlibTest {
}
@Test
@Ignore("TODO(bytecode-only): range take mismatch")
fun testTake() = runTest {
eval("""
val r = 1..8
@ -52,7 +50,6 @@ class StdlibTest {
}
@Test
@Ignore("TODO(bytecode-only): any/all mismatch")
fun testAnyAndAll() = runTest {
eval("""
assert( [1,2,3].any { it > 2 } )
@ -90,7 +87,6 @@ class StdlibTest {
}
@Test
@Ignore("TODO(bytecode-only): range drop mismatch")
fun testDrop() = runTest {
eval("""
assertEquals([7,8], (1..8).drop(6).toList() )