SergeychWorks/lyng
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases 14 Wiki Activity

refs #36 imports refined and optimized, circular imports are supported.

Browse Source
This commit is contained in:
Sergey Chernov 2025-07-07 15:01:27 +03:00
parent 1e2cb5e420
commit ef6bc5c468
9 changed files with 227 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt
View File

@ -1,11 +1,13 @@
package net.sergeych.lyng
import net.sergeych.lyng.pacman.ImportProvider
/**
* The LYNG compiler.
*/
class Compiler(
val cc: CompilerContext,
val pacman: Pacman = Pacman.emptyAllowAll,
val importProvider: ImportProvider = ImportProvider.emptyAllowAll,
@Suppress("UNUSED_PARAMETER")
settings: Settings = Settings()
) {
@ -41,9 +43,9 @@ class Compiler(
cc.next()
val pos = cc.currentPos()
val name = loadQualifiedName()
pacman.prepareImport(pos, name, null)
val module = importProvider.prepareImport(pos, name, null)
statements += statement {
pacman.performImport(this,name,null)
module.importInto(this, null)
ObjVoid
}
continue
@ -1618,8 +1620,8 @@ class Compiler(
companion object {
suspend fun compile(source: Source,pacman: Pacman = Pacman.emptyAllowAll): Script {
return Compiler(CompilerContext(parseLyng(source)),pacman).parseScript()
suspend fun compile(source: Source, importProvider: ImportProvider = ImportProvider.emptyAllowAll): Script {
return Compiler(CompilerContext(parseLyng(source)),importProvider).parseScript()
}
private var lastPriority = 0

48
lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt
View File

@ -1,48 +0,0 @@
package net.sergeych.lyng
import kotlinx.coroutines.CompletableDeferred
import net.sergeych.mp_tools.globalDefer
import net.sergeych.mp_tools.globalLaunch
/**
* Naive source-based pacman that compiles all sources first, before first import could be resolved.
* It supports imports between [sources] but does not resolve nor detect cyclic imports which
* are not supported.
*/
class InlineSourcesPacman(pacman: Pacman, val sources: List<Source>) : Pacman(pacman) {
data class Entry(val source: Source, val deferredModule: CompletableDeferred<ModuleScope> = CompletableDeferred())
private val modules = run {
val result = mutableMapOf<String, Entry>()
for (source in sources) {
val name = source.extractPackageName()
result[name] = Entry(source)
}
val inner = InitMan()
for ((name, entry) in result) {
globalLaunch {
val scope = ModuleScope(inner, entry.source.startPos, name)
Compiler.compile(entry.source, inner).execute(scope)
entry.deferredModule.complete(scope)
}
}
result
}
inner class InitMan : Pacman(parent) {
override suspend fun createModuleScope(name: String): ModuleScope? = modules[name]?.deferredModule?.await()
}
private val readyModules by lazy {
globalDefer {
modules.entries.map { it.key to it.value.deferredModule.await() }.toMap()
}
}
override suspend fun createModuleScope(name: String): ModuleScope? =
readyModules.await()[name]
}

44
lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt
View File

@ -1,33 +1,47 @@
package net.sergeych.lyng
import net.sergeych.lyng.pacman.ImportProvider
/**
* Module scope supports importing and contains the [pacman]; it should be the same
* Module scope supports importing and contains the [importProvider]; it should be the same
* used in [Compiler];
*/
class ModuleScope(
val pacman: Pacman,
val importProvider: ImportProvider,
pos: Pos = Pos.builtIn,
val packageName: String
) : Scope(pacman.rootScope, Arguments.EMPTY, pos) {
override val packageName: String
) : Scope(importProvider.rootScope, Arguments.EMPTY, pos) {
constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName)
override suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>?) {
pacman.prepareImport(pos, name, symbols)
}
constructor(importProvider: ImportProvider, source: Source) : this(importProvider, source.startPos, source.fileName)
/**
* Import symbols into the scope. It _is called_ after the module is imported by [checkImport].
* If [checkImport] was not called, the symbols will not be imported with exception as module is not found.
* Import symbols into the scope. It _is called_ after the module is imported by [ImportProvider.prepareImport]
* which checks symbol availability and accessibility prior to execution.
* @param scope where to copy symbols from this module
* @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
*/
override suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>?) {
pacman.performImport(scope, name, symbols)
override suspend fun importInto(scope: Scope, symbols: Map<String, String>?) {
val symbolsToImport = symbols?.keys?.toMutableSet()
for ((symbol, record) in this.objects) {
if (record.visibility.isPublic) {
val newName = symbols?.let { ss: Map<String, String> ->
ss[symbol]
?.also { symbolsToImport!!.remove(it) }
?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
} ?: symbol
if (newName in scope.objects)
scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
scope.objects[newName] = record
}
}
if (!symbolsToImport.isNullOrEmpty())
scope.raiseSymbolNotFound("symbols $packageName.{$symbolsToImport} are.were not found")
}
val packageNameObj by lazy { ObjString(packageName).asReadonly}
val packageNameObj by lazy { ObjString(packageName).asReadonly }
override fun get(name: String): ObjRecord? {
return if( name == "__PACKAGE__")
return if (name == "__PACKAGE__")
packageNameObj
else
super.get(name)

69
lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt
View File

@ -1,69 +0,0 @@
package net.sergeych.lyng
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
/**
* Package manager. Chained manager, too simple. Override [createModuleScope] to return either
* valid [ModuleScope] or call [parent] - or return null.
*/
abstract class Pacman(
val parent: Pacman? = null,
val rootScope: Scope = parent!!.rootScope,
val securityManager: SecurityManager = parent!!.securityManager
) {
private val opScopes = Mutex()
private val cachedScopes = mutableMapOf<String, Scope>()
/**
* Create a new module scope if this pacman can import the module, return null otherwise so
* the manager can decide what to do
*/
abstract suspend fun createModuleScope(name: String): ModuleScope?
suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?) {
if (!securityManager.canImportModule(name))
throw ImportException(pos, "Module $name is not allowed")
symbols?.keys?.forEach {
if (!securityManager.canImportSymbol(name, it)) throw ImportException(
pos,
"Symbol $name.$it is not allowed"
)
}
// if we can import the module, cache it, or go further
opScopes.withLock {
cachedScopes[name] ?: createModuleScope(name)?.let { cachedScopes[name] = it }
} ?: parent?.prepareImport(pos, name, symbols) ?: throw ImportException(pos, "Module $name is not found")
}
suspend fun performImport(scope: Scope, name: String, symbols: Map<String, String>?) {
val module = opScopes.withLock { cachedScopes[name] }
?: scope.raiseSymbolNotFound("module $name not found")
val symbolsToImport = symbols?.keys?.toMutableSet()
for ((symbol, record) in module.objects) {
if (record.visibility.isPublic) {
val newName = symbols?.let { ss: Map<String, String> ->
ss[symbol]
?.also { symbolsToImport!!.remove(it) }
?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
} ?: symbol
if (newName in scope.objects)
scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
scope.objects[newName] = record
}
}
if (!symbolsToImport.isNullOrEmpty())
scope.raiseSymbolNotFound("symbols $name.{$symbolsToImport} are.were not found")
}
companion object {
val emptyAllowAll = object : Pacman(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
override suspend fun createModuleScope(name: String): ModuleScope? {
return null
}
}
}
}

21
lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt
View File

@ -1,5 +1,7 @@
package net.sergeych.lyng
import net.sergeych.lyng.pacman.ImportProvider
/**
* Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes.
* Each block usually creates a scope. Accessing Lyng closures usually is done via a scope.
@ -16,6 +18,8 @@ open class Scope(
var thisObj: Obj = ObjVoid,
var skipScopeCreation: Boolean = false,
) {
open val packageName: String = "<anonymous package>"
constructor(
args: Arguments = Arguments.EMPTY,
pos: Pos = Pos.builtIn,
@ -137,16 +141,17 @@ open class Scope(
suspend fun eval(source: Source): Obj =
Compiler.compile(
source,
(this as? ModuleScope)?.pacman ?: Pacman.emptyAllowAll
).execute(this)
(this as? ModuleScope)?.importProvider ?: ImportProvider.emptyAllowAll
).execute(this)
fun containsLocal(name: String): Boolean = name in objects
open suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>? = null) {
throw ImportException(pos, "Import is not allowed here: $name")
}
open suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>? = null) {
scope.raiseError(ObjIllegalOperationException(scope,"Import is not allowed here: import $name"))
/**
* Some scopes can be imported into other scopes, like [ModuleScope]. Those must correctly implement this method.
* @param scope where to copy symbols from this module
* @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
*/
open suspend fun importInto(scope: Scope, symbols: Map<String, String>? = null) {
scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName"))
}
}

48
lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt Normal file
View File

@ -0,0 +1,48 @@
package net.sergeych.lyng.pacman
import net.sergeych.lyng.*
/**
* Package manager INTERFACE (abstract class). Performs import routines
* using abstract [createModuleScope] method ot be implemented by heirs.
*
* Notice that [createModuleScope] is responsible for caching the modules;
* base class relies on caching. This is not implemented here as the correct
* caching strategy depends on the import provider
*/
abstract class ImportProvider(
val rootScope: Scope = Script.defaultScope,
val securityManager: SecurityManager = SecurityManager.allowAll
) {
/**
* Find an import and create a scope for it. This method must implement caching so repeated
* imports are not repeatedly loaded and parsed and should be cheap.
*
* @throws ImportException if the module is not found
*/
abstract suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope
/**
* Check that the import is possible and allowed. This method is called on compile time by [Compiler];
* actual module loading is performed by [ModuleScope.importInto]
*/
suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?): ModuleScope {
if (!securityManager.canImportModule(name))
throw ImportException(pos, "Module $name is not allowed")
symbols?.keys?.forEach {
if (!securityManager.canImportSymbol(name, it)) throw ImportException(
pos,
"Symbol $name.$it is not allowed"
)
}
return createModuleScope(pos, name)
}
companion object {
val emptyAllowAll = object : ImportProvider(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
throw ImportException(pos, "Empty import provider can't be used directly")
}
}
}
}

63
lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt Normal file
View File

@ -0,0 +1,63 @@
package net.sergeych.lyng.pacman
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import net.sergeych.lyng.*
/**
* The import provider that imports sources available in memory.
*/
class InlineSourcesImportProvider(val sources: List<Source>,
rootScope: ModuleScope = ModuleScope(emptyAllowAll, Source.builtIn),
securityManager: SecurityManager = SecurityManager.allowAll
) : ImportProvider(rootScope, securityManager) {
private class Entry(
val source: Source,
var scope: ModuleScope? = null,
)
private val inner = InitMan()
private val modules = run {
val result = mutableMapOf<String, Entry>()
for (source in sources) {
val name = source.extractPackageName()
result[name] = Entry(source)
}
result
}
private var access = Mutex()
/**
* Inner provider does not lock [access], the only difference; it is meant to be used
* exclusively by the coroutine that starts actual import chain
*/
private inner class InitMan : ImportProvider() {
override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
return doImport(packageName, pos)
}
}
/**
* External interface, thread-safe. Can suspend until actual import is done. implements caching.
*/
override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope =
access.withLock {
doImport(packageName, pos)
}
/**
* Perform actual import or return ready scope. __It must only be called when
* [access] is locked__, e.g. only internally
*/
private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
modules[packageName]?.scope?.let { return it }
val entry = modules[packageName] ?: throw ImportException(pos, "Unknown package $packageName")
return ModuleScope(inner, pos, packageName).apply {
entry.scope = this
eval(entry.source)
}
}
}

78
lynglib/src/commonTest/kotlin/ScriptTest.kt
View File

@ -1,8 +1,8 @@
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.test.runTest
import net.sergeych.lyng.*
import net.sergeych.lyng.pacman.InlineSourcesImportProvider
import kotlin.test.*
class ScriptTest {
@ -2250,18 +2250,21 @@ class ScriptTest {
@Test
fun testLet() = runTest {
eval("""
eval(
"""
class Point(x=0,y=0)
assert( Point() is Object)
Point().let { println(it.x, it.y) }
val x = null
x?.let { println(it.x, it.y) }
""".trimIndent())
""".trimIndent()
)
}
@Test
fun testApply() = runTest {
eval("""
eval(
"""
class Point(x,y)
// see the difference: apply changes this to newly created Point:
val p = Point(1,2).apply {
@ -2270,12 +2273,14 @@ class ScriptTest {
assertEquals(p, Point(2,3))
>>> void
""".trimIndent())
""".trimIndent()
)
}
@Test
fun testApplyThis() = runTest {
eval("""
eval(
"""
class Point(x,y)
// see the difference: apply changes this to newly created Point:
val p = Point(1,2).apply {
@ -2284,12 +2289,14 @@ class ScriptTest {
assertEquals(p, Point(2,3))
>>> void
""".trimIndent())
""".trimIndent()
)
}
@Test
fun testExtend() = runTest() {
eval("""
eval(
"""
fun Int.isEven() {
this % 2 == 0
@ -2311,7 +2318,8 @@ class ScriptTest {
assert( 12.1.isInteger() == false )
assert( "5".isInteger() )
assert( ! "5.2".isInteger() )
""".trimIndent())
""".trimIndent()
)
}
@Test
@ -2319,18 +2327,20 @@ class ScriptTest {
val c = Scope()
val arr = c.eval("[1,2,3]")
// array is iterable so we can:
assertEquals(listOf(1,2,3), arr.toFlow(c).map { it.toInt() }.toList())
assertEquals(listOf(1, 2, 3), arr.toFlow(c).map { it.toInt() }.toList())
}
@Test
fun testAssociateBy() = runTest() {
eval("""
eval(
"""
val m = [123, 456].associateBy { "k:%s"(it) }
println(m)
assertEquals(123, m["k:123"])
assertEquals(456, m["k:456"])
""")
listOf(1,2,3).associateBy { it * 10 }
"""
)
listOf(1, 2, 3).associateBy { it * 10 }
}
// @Test
@ -2354,7 +2364,7 @@ class ScriptTest {
fun foo() { "foo1" }
""".trimIndent()
val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
val pm = InlineSourcesImportProvider(listOf(Source("foosrc", foosrc)))
val src = """
import lyng.foo
@ -2380,11 +2390,12 @@ class ScriptTest {
fun bar() { "bar1" }
""".trimIndent()
val pm = InlineSourcesPacman(
Pacman.emptyAllowAll, listOf(
val pm = InlineSourcesImportProvider(
listOf(
Source("barsrc", barsrc),
Source("foosrc", foosrc),
))
)
)
val src = """
import lyng.foo
@ -2396,4 +2407,37 @@ class ScriptTest {
assertEquals("foo1 / bar1", scope.eval(src).toString())
}
@Test
fun testImportsCircular() = runTest {
val foosrc = """
package lyng.foo
import lyng.bar
fun foo() { "foo1" }
""".trimIndent()
val barsrc = """
package lyng.bar
import lyng.foo
fun bar() { "bar1" }
""".trimIndent()
val pm = InlineSourcesImportProvider(
listOf(
Source("barsrc", barsrc),
Source("foosrc", foosrc),
)
)
val src = """
import lyng.bar
foo() + " / " + bar()
""".trimIndent().toSource("test")
val scope = ModuleScope(pm, src)
assertEquals("foo1 / bar1", scope.eval(src).toString())
}
}

9
lynglib/src/jvmTest/kotlin/OtherTests.kt
View File

@ -1,6 +1,9 @@
import junit.framework.TestCase.assertEquals
import kotlinx.coroutines.runBlocking
import net.sergeych.lyng.*
import net.sergeych.lyng.ModuleScope
import net.sergeych.lyng.Source
import net.sergeych.lyng.pacman.InlineSourcesImportProvider
import net.sergeych.lyng.toSource
import kotlin.test.Test
class OtherTests {
@ -18,8 +21,8 @@ class OtherTests {
fun bar() { "bar1" }
""".trimIndent()
val pm = InlineSourcesPacman(
Pacman.emptyAllowAll, listOf(
val pm = InlineSourcesImportProvider(
listOf(
Source("foosrc", foosrc),
Source("barsrc", barsrc),
))