Fix JVM import caching and class/object bytecode dispatch

examples/fillspeed.lyng

@@ -0,0 +1,15 @@
+import lyng.time
+
+val n = 700_000
+
+fun tm<T>(block: ()->T): T {
+    val t = Instant()
+    block().also {
+        println("tm: ${Instant() - t}")
+    }
+}
+
+val x = tm { List.fill(n) { it * 10 + 1 } }
+val y = tm { List.fill(n, n + 10) { it * 10 + 1 } }
+tm { x.add(-1) }
+tm { y.add(-2) }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1948,7 +1948,7 @@ class Compiler(
                         slotTypeDeclByScopeId = slotTypeDeclByScopeId,
                         knownNameObjClass = knownClassMapForBytecode(),
                         knownClassNames = knownClassNamesForBytecode(),
-                        knownObjectNames = objectDeclNames,
+                        knownObjectNames = knownObjectNamesForBytecode(),
                         classFieldTypesByName = classFieldTypesByName,
                         enumEntriesByName = enumEntriesByName,
                         callableReturnTypeByScopeId = callableReturnTypeByScopeId,
@@ -2277,6 +2277,23 @@ class Compiler(
         return result
     }
 
+    private fun knownObjectNamesForBytecode(): Set<String> {
+        val result = LinkedHashSet<String>()
+        fun addScope(scope: Scope?) {
+            if (scope == null) return
+            for ((name, rec) in scope.objects) {
+                if (rec.value is ObjInstance) result.add(name)
+            }
+        }
+        addScope(seedScope)
+        addScope(importManager.rootScope)
+        for (module in importedModules) {
+            addScope(module.scope)
+        }
+        result.addAll(objectDeclNames)
+        return result
+    }
+
     private fun wrapBytecode(stmt: Statement): Statement {
         if (codeContexts.lastOrNull() is CodeContext.Module) return stmt
         if (codeContexts.lastOrNull() is CodeContext.ClassBody) return stmt
@@ -2305,7 +2322,7 @@ class Compiler(
             slotTypeDeclByScopeId = slotTypeDeclByScopeId,
             knownNameObjClass = knownClassMapForBytecode(),
             knownClassNames = knownClassNamesForBytecode(),
-            knownObjectNames = objectDeclNames,
+            knownObjectNames = knownObjectNamesForBytecode(),
             classFieldTypesByName = classFieldTypesByName,
             enumEntriesByName = enumEntriesByName,
             callableReturnTypeByScopeId = callableReturnTypeByScopeId,
@@ -2340,7 +2357,7 @@ class Compiler(
             slotTypeDeclByScopeId = slotTypeDeclByScopeId,
             knownNameObjClass = knownClassMapForBytecode(),
             knownClassNames = knownClassNamesForBytecode(),
-            knownObjectNames = objectDeclNames,
+            knownObjectNames = knownObjectNamesForBytecode(),
             classFieldTypesByName = classFieldTypesByName,
             enumEntriesByName = enumEntriesByName,
             callableReturnTypeByScopeId = callableReturnTypeByScopeId,
@@ -2400,7 +2417,7 @@ class Compiler(
             slotTypeDeclByScopeId = slotTypeDeclByScopeId,
             knownNameObjClass = knownNames,
             knownClassNames = knownClassNamesForBytecode(),
-            knownObjectNames = objectDeclNames,
+            knownObjectNames = knownObjectNamesForBytecode(),
             classFieldTypesByName = classFieldTypesByName,
             enumEntriesByName = enumEntriesByName,
             callableReturnTypeByScopeId = callableReturnTypeByScopeId,

lynglib/src/commonMain/kotlin/net/sergeych/lyng/bytecode/BytecodeCompiler.kt

@@ -2688,9 +2688,7 @@ class BytecodeCompiler(
                 }
                 return value
             }
-            if ((isKnownClassReceiver(target.target) || isClassNameRef(target.target, receiverClass)) &&
+            if (shouldUseClassScopeReceiver(target.target, receiver, receiverClass)) {
-                (isClassSlot(receiver.slot) || receiverClass == ObjClassType)
-            ) {
                 val nameId = builder.addConst(BytecodeConst.StringVal(target.name))
                 if (!target.isOptional) {
                     builder.emit(Opcode.SET_CLASS_SCOPE, receiver.slot, nameId, value.slot)
@@ -3050,7 +3048,7 @@ class BytecodeCompiler(
             }
             val fieldId = if (resolvedMember != null) receiverClass.instanceFieldIdMap()[fieldTarget.name] else null
             val methodId = if (resolvedMember != null) receiverClass.instanceMethodIdMap(includeAbstract = true)[fieldTarget.name] else null
-            if (fieldId == null && methodId == null && (receiverClass == ObjClassType || isKnownClassReceiver(fieldTarget.target) || isClassNameRef(fieldTarget.target, receiverClass))) {
+            if (fieldId == null && methodId == null && shouldUseClassScopeReceiver(fieldTarget.target, receiver, receiverClass)) {
                 val nameId = builder.addConst(BytecodeConst.StringVal(fieldTarget.name))
                 if (!fieldTarget.isOptional) {
                     builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, current)
@@ -3564,9 +3562,7 @@ class BytecodeCompiler(
         val encodedMethodId = encodeMemberId(receiverClass, methodId)
         val receiver = compileRefWithFallback(ref.target, null, pos) ?: return null
         val dst = allocSlot()
-        if (fieldId == null && methodId == null && (isKnownClassReceiver(ref.target) || isClassNameRef(ref.target, receiverClass)) &&
+        if (fieldId == null && methodId == null && shouldUseClassScopeReceiver(ref.target, receiver, receiverClass)) {
-            (isClassSlot(receiver.slot) || receiverClass == ObjClassType)
-        ) {
             val nameId = builder.addConst(BytecodeConst.StringVal(ref.name))
             if (!ref.isOptional) {
                 builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, dst)
@@ -4244,9 +4240,7 @@ class BytecodeCompiler(
             }
             val fieldId = receiverClass.instanceFieldIdMap()[fieldTarget.name]
             val methodId = receiverClass.instanceMethodIdMap(includeAbstract = true)[fieldTarget.name]
-            if (fieldId == null && methodId == null && (isKnownClassReceiver(fieldTarget.target) || isClassNameRef(fieldTarget.target, receiverClass)) &&
+            if (fieldId == null && methodId == null && shouldUseClassScopeReceiver(fieldTarget.target, receiver, receiverClass)) {
-                (isClassSlot(receiver.slot) || receiverClass == ObjClassType)
-            ) {
                 val nameId = builder.addConst(BytecodeConst.StringVal(fieldTarget.name))
                 val resultSlot = allocSlot()
                 if (fieldTarget.isOptional) {
@@ -4806,6 +4800,34 @@ class BytecodeCompiler(
         val receiverClass = resolveReceiverClass(ref.receiver) ?: ObjDynamic.type
         val receiver = compileRefWithFallback(ref.receiver, null, refPosOrCurrent(ref.receiver)) ?: return null
         val dst = allocSlot()
+        if (shouldUseClassScopeReceiver(ref.receiver, receiver, receiverClass)) {
+            val nameId = builder.addConst(BytecodeConst.StringVal(ref.name))
+            val memberSlot = allocSlot()
+            if (!ref.isOptional) {
+                builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, memberSlot)
+            } else {
+                val nullSlot = allocSlot()
+                builder.emit(Opcode.CONST_NULL, nullSlot)
+                val cmpSlot = allocSlot()
+                builder.emit(Opcode.CMP_REF_EQ_OBJ, receiver.slot, nullSlot, cmpSlot)
+                val nullLabel = builder.label()
+                val endLabel = builder.label()
+                builder.emit(
+                    Opcode.JMP_IF_TRUE,
+                    listOf(CmdBuilder.Operand.IntVal(cmpSlot), CmdBuilder.Operand.LabelRef(nullLabel))
+                )
+                builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, memberSlot)
+                builder.emit(Opcode.JMP, listOf(CmdBuilder.Operand.LabelRef(endLabel)))
+                builder.mark(nullLabel)
+                builder.emit(Opcode.CONST_NULL, memberSlot)
+                builder.mark(endLabel)
+            }
+            val args = compileCallArgs(ref.args, ref.tailBlock, ref.explicitTypeArgs) ?: return null
+            val encodedCount = encodeCallArgCount(args) ?: return null
+            setPos(callPos)
+            emitCallCompiled(CompiledValue(memberSlot, SlotType.OBJ), args.base, encodedCount, dst)
+            return CompiledValue(dst, SlotType.OBJ)
+        }
         fun emitDynamicCall(): CompiledValue? {
             val args = compileCallArgs(ref.args, ref.tailBlock, ref.explicitTypeArgs) ?: return null
             val encodedCount = encodeCallArgCount(args) ?: return null
@@ -4918,34 +4940,6 @@ class BytecodeCompiler(
             }
             return CompiledValue(dst, SlotType.OBJ)
         }
-        if (isKnownClassReceiver(ref.receiver)) {
-            val nameId = builder.addConst(BytecodeConst.StringVal(ref.name))
-            val memberSlot = allocSlot()
-            if (!ref.isOptional) {
-                builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, memberSlot)
-            } else {
-                val nullSlot = allocSlot()
-                builder.emit(Opcode.CONST_NULL, nullSlot)
-                val cmpSlot = allocSlot()
-                builder.emit(Opcode.CMP_REF_EQ_OBJ, receiver.slot, nullSlot, cmpSlot)
-                val nullLabel = builder.label()
-                val endLabel = builder.label()
-                builder.emit(
-                    Opcode.JMP_IF_TRUE,
-                    listOf(CmdBuilder.Operand.IntVal(cmpSlot), CmdBuilder.Operand.LabelRef(nullLabel))
-                )
-                builder.emit(Opcode.GET_CLASS_SCOPE, receiver.slot, nameId, memberSlot)
-                builder.emit(Opcode.JMP, listOf(CmdBuilder.Operand.LabelRef(endLabel)))
-                builder.mark(nullLabel)
-                builder.emit(Opcode.CONST_NULL, memberSlot)
-                builder.mark(endLabel)
-            }
-            val args = compileCallArgs(ref.args, ref.tailBlock, ref.explicitTypeArgs) ?: return null
-            val encodedCount = encodeCallArgCount(args) ?: return null
-            setPos(callPos)
-            emitCallCompiled(CompiledValue(memberSlot, SlotType.OBJ), args.base, encodedCount, dst)
-            return CompiledValue(dst, SlotType.OBJ)
-        }
         val extSlot = resolveExtensionCallableSlot(receiverClass, ref.name)
             ?: throw BytecodeCompileException(
                 missingMemberMessage(receiverClass, ref.name),
@@ -5544,7 +5538,7 @@ class BytecodeCompiler(
         if (scopeSlotCount > 0) {
             for (index in 0 until scopeSlotCount) {
                 val name = scopeSlotNames.getOrNull(index) ?: continue
-                resolveTypeNameClass(name)?.let { trackExactCallableObjAtSlot(index, it) }
+                preloadExactCallableName(index, name)
             }
         }
         if (!allowLocalSlots || localSlotNames.isEmpty()) return
@@ -5552,12 +5546,17 @@ class BytecodeCompiler(
             val name = localSlotNames[localIndex] ?: continue
             val key = localSlotKeyByIndex.getOrNull(localIndex)
             val isModuleLocal = key != null && moduleScopeId != null && key.scopeId == moduleScopeId
-            val isCapture = localSlotCaptures.getOrNull(localIndex) == true
+            val isCapture = key != null && captureSlotKeys.contains(key)
             if (!isModuleLocal && !isCapture) continue
-            resolveTypeNameClass(name)?.let { trackExactCallableObjAtSlot(scopeSlotCount + localIndex, it) }
+            preloadExactCallableName(scopeSlotCount + localIndex, name)
         }
     }
 
+    private fun preloadExactCallableName(slot: Int, name: String) {
+        if (slotObjClass[slot] != ObjClassType && callSignatureByName[name] == null) return
+        resolveTypeNameClass(name)?.let { trackExactCallableObjAtSlot(slot, it) }
+    }
+
     private fun collectExactLambdaModuleCaptures() {
         if (exactLambdaRefByScopeId.isEmpty()) return
         val seen = LinkedHashSet<LambdaFnRef>()
@@ -8324,39 +8323,69 @@ class BytecodeCompiler(
         return when (ref) {
             is LocalVarRef -> {
                 val name = ref.name
-                if (nameObjClass[name] == ObjClassType) return true
                 val directSlot = resolveDirectNameSlot(name)?.slot
-                if (directSlot != null && slotObjClass[directSlot] == ObjClassType) return true
+                if (directSlot != null) return isNamedClassBinding(name, directSlot)
+                if (nameObjClass[name] == ObjClassType) return true
                 if (localSlotIndexByName.containsKey(name)) return false
                 if (localSlotInfoMap.values.any { it.name == name }) return false
-                (knownClassNames.contains(name) || classFieldTypesByName.containsKey(name)) &&
+                isUnboundClassName(name)
-                    !knownObjectNames.contains(name)
             }
             is LocalSlotRef -> {
                 val name = ref.name
-                if (slotObjClass[ref.slot] == ObjClassType) return true
+                if (slotObjClass[ref.slot] != null || exactCallableObjBySlot[ref.slot] != null) {
-                if (nameObjClass[name] == ObjClassType) return true
+                    return isNamedClassBinding(name, ref.slot)
+                }
                 val directSlot = resolveDirectNameSlot(name)?.slot
-                if (directSlot != null && slotObjClass[directSlot] == ObjClassType) return true
+                if (directSlot != null) return isNamedClassBinding(name, directSlot)
+                if (nameObjClass[name] == ObjClassType) return true
                 if (localSlotIndexByName.containsKey(name)) return false
                 if (localSlotInfoMap.values.any { it.name == name }) return false
-                (knownClassNames.contains(name) || classFieldTypesByName.containsKey(name)) &&
+                isUnboundClassName(name)
-                    !knownObjectNames.contains(name)
             }
             is FastLocalVarRef -> {
                 val name = ref.name
-                if (nameObjClass[name] == ObjClassType) return true
                 val directSlot = resolveDirectNameSlot(name)?.slot
-                if (directSlot != null && slotObjClass[directSlot] == ObjClassType) return true
+                if (directSlot != null) return isNamedClassBinding(name, directSlot)
+                if (nameObjClass[name] == ObjClassType) return true
                 if (localSlotIndexByName.containsKey(name)) return false
                 if (localSlotInfoMap.values.any { it.name == name }) return false
-                (knownClassNames.contains(name) || classFieldTypesByName.containsKey(name)) &&
+                isUnboundClassName(name)
-                    !knownObjectNames.contains(name)
             }
             else -> false
         }
     }
 
+    private fun shouldUseClassScopeReceiver(
+        ref: ObjRef,
+        receiver: CompiledValue,
+        receiverClass: ObjClass
+    ): Boolean {
+        val name = when (ref) {
+            is LocalVarRef -> ref.name
+            is LocalSlotRef -> ref.name
+            is FastLocalVarRef -> ref.name
+            else -> null
+        }
+        if (name != null && isKnownObjectBinding(name)) return false
+        return isKnownClassReceiver(ref) &&
+            (isClassSlot(receiver.slot) || receiverClass == ObjClassType || isClassNameRef(ref, receiverClass))
+    }
+
+    private fun isNamedClassBinding(name: String, slot: Int): Boolean {
+        if (isClassSlot(slot)) return true
+        if (isKnownObjectBinding(name)) return false
+        return knownClassNames.contains(name) || classFieldTypesByName.containsKey(name)
+    }
+
+    private fun isUnboundClassName(name: String): Boolean {
+        if (isKnownObjectBinding(name)) return false
+        return knownClassNames.contains(name) || classFieldTypesByName.containsKey(name)
+    }
+
+    private fun isKnownObjectBinding(name: String): Boolean {
+        return knownObjectNames.contains(name)
+    }
+
     private fun isClassNameRef(ref: ObjRef, receiverClass: ObjClass): Boolean {
         if (receiverClass !is ObjInstanceClass) return false
         val name = when (ref) {
@@ -8365,10 +8394,11 @@ class BytecodeCompiler(
             is FastLocalVarRef -> ref.name
             else -> return false
         }
-        return name == receiverClass.className
+        return name == receiverClass.className && isKnownClassReceiver(ref)
     }
 
-    private fun isClassSlot(slot: Int): Boolean = slotObjClass[slot] == ObjClassType
+    private fun isClassSlot(slot: Int): Boolean =
+        slotObjClass[slot] == ObjClassType || exactCallableObjBySlot[slot] is ObjClass
 
 
     private fun isThisReceiver(ref: ObjRef): Boolean {
@@ -9320,8 +9350,10 @@ class BytecodeCompiler(
                 val localIndex = localSlotIndexByKey[key] ?: continue
                 val slot = scopeSlotCount + localIndex
                 localSlotCaptures[localIndex] = true
-                forcedObjSlots.add(slot)
+                if (!slotTypes.containsKey(slot)) {
-                slotTypes[slot] = SlotType.OBJ
+                    forcedObjSlots.add(slot)
+                    slotTypes[slot] = SlotType.OBJ
+                }
             }
         }
         if (allowLocalSlots && valueFnRefs.isNotEmpty() && lambdaCaptureEntriesByRef.isNotEmpty()) {
@@ -9349,8 +9381,10 @@ class BytecodeCompiler(
                     val localIndex = localSlotIndexByName[name] ?: continue
                     val slot = scopeSlotCount + localIndex
                     localSlotCaptures[localIndex] = true
-                    forcedObjSlots.add(slot)
+                    if (!slotTypes.containsKey(slot)) {
-                    slotTypes[slot] = SlotType.OBJ
+                        forcedObjSlots.add(slot)
+                        slotTypes[slot] = SlotType.OBJ
+                    }
                 }
             }
         }
@@ -9383,6 +9417,7 @@ class BytecodeCompiler(
                 }
             }
         }
+        preloadExactCallableNames()
         if (loopVarKeys.isNotEmpty()) {
             for (key in loopVarKeys) {
                 val localIndex = localSlotIndexByKey[key]
@@ -9460,11 +9495,10 @@ class BytecodeCompiler(
                         }
                         if (!scopeSlotNameMap.containsKey(key)) {
                             scopeSlotNameMap[key] = stmt.spec.name
+                        }
+                    }
                 }
             }
-        }
-        preloadExactCallableNames()
-    }
             is DelegatedVarDeclStatement -> {
                 val slotIndex = stmt.slotIndex
                 val scopeId = stmt.scopeId ?: 0

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt

@@ -38,17 +38,26 @@ class ImportManager(
 
     val packageNames: List<String> get() = imports.keys.toList()
 
+    private class CacheCell(var scope: ModuleScope? = null)
+
     private inner class Entry(
         val packageName: String,
         val builder: suspend (ModuleScope) -> Unit,
-        var cachedScope: ModuleScope? = null
+        val cacheCell: CacheCell = CacheCell()
     ) {
 
         suspend fun getScope(pos: Pos): ModuleScope {
-            cachedScope?.let { return it }
+            cacheCell.scope?.let { return it }
-            return ModuleScope(inner, pos, packageName).apply {
+            val module = ModuleScope(inner, pos, packageName)
-                cachedScope = this
+            cacheCell.scope = module
-                builder(this)
+            return try {
+                builder(module)
+                module
+            } catch (e: Throwable) {
+                if (cacheCell.scope === module) {
+                    cacheCell.scope = null
+                }
+                throw e
             }
         }
     }
@@ -152,14 +161,14 @@ class ImportManager(
         op.withLock {
             ImportManager(rootScope, securityManager).apply {
                 for ((name, entry) in this@ImportManager.imports) {
-                    imports[name] = Entry(entry.packageName, entry.builder, entry.cachedScope)
+                    imports[name] = Entry(entry.packageName, entry.builder, entry.cacheCell)
                 }
             }
         }
 
     fun invalidatePackageCache(name: String) {
         op.withLock {
-            imports[name]?.cachedScope = null
+            imports[name]?.cacheCell?.scope = null
         }
     }
 

lynglib/src/commonTest/kotlin/BytecodeRecentOpsTest.kt

@@ -247,6 +247,21 @@ class BytecodeRecentOpsTest {
         assertEquals(11, scope.eval("calc()").toInt())
     }
 
+    @Test
+    fun capturedLambdaCanCallListFillOnCapturedClassReceiver() = runTest {
+        val scope = Script.newScope()
+        val result = scope.eval(
+            """
+            fun calc(n: Int) {
+                val xs = { List.fill(n) { it } }()
+                xs[4]
+            }
+            calc(5)
+            """.trimIndent()
+        )
+        assertEquals(4, result.toInt())
+    }
+
     @Test
     fun directLambdaLiteralCallWithCaptureUsesInlineBytecode() = runTest {
         val scope = Script.newScope()