refs #36 imports refined and optimized, circular imports are supported.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1,11 +1,13 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.pacman.ImportProvider
+
 /**
  * The LYNG compiler.
  */
 class Compiler(
     val cc: CompilerContext,
-    val pacman: Pacman = Pacman.emptyAllowAll,
+    val importProvider: ImportProvider = ImportProvider.emptyAllowAll,
     @Suppress("UNUSED_PARAMETER")
     settings: Settings = Settings()
 ) {
@@ -41,9 +43,9 @@ class Compiler(
                         cc.next()
                         val pos = cc.currentPos()
                         val name = loadQualifiedName()
-                        pacman.prepareImport(pos, name, null)
+                        val module = importProvider.prepareImport(pos, name, null)
                         statements += statement {
-                            pacman.performImport(this,name,null)
+                            module.importInto(this, null)
                             ObjVoid
                         }
                         continue
@@ -1618,8 +1620,8 @@ class Compiler(
 
     companion object {
 
-        suspend fun compile(source: Source,pacman: Pacman = Pacman.emptyAllowAll): Script {
+        suspend fun compile(source: Source, importProvider: ImportProvider = ImportProvider.emptyAllowAll): Script {
-            return Compiler(CompilerContext(parseLyng(source)),pacman).parseScript()
+            return Compiler(CompilerContext(parseLyng(source)),importProvider).parseScript()
         }
 
         private var lastPriority = 0

lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt

@@ -1,48 +0,0 @@
-package net.sergeych.lyng
-
-import kotlinx.coroutines.CompletableDeferred
-import net.sergeych.mp_tools.globalDefer
-import net.sergeych.mp_tools.globalLaunch
-
-/**
- * Naive source-based pacman that compiles all sources first, before first import could be resolved.
- * It supports imports between [sources] but does not resolve nor detect cyclic imports which
- * are not supported.
- */
-class InlineSourcesPacman(pacman: Pacman, val sources: List<Source>) : Pacman(pacman) {
-
-    data class Entry(val source: Source, val deferredModule: CompletableDeferred<ModuleScope> = CompletableDeferred())
-
-
-    private val modules = run {
-        val result = mutableMapOf<String, Entry>()
-        for (source in sources) {
-            val name = source.extractPackageName()
-            result[name] = Entry(source)
-        }
-        val inner = InitMan()
-
-        for ((name, entry) in result) {
-            globalLaunch {
-                val scope = ModuleScope(inner, entry.source.startPos, name)
-                Compiler.compile(entry.source, inner).execute(scope)
-                entry.deferredModule.complete(scope)
-            }
-        }
-        result
-    }
-
-    inner class InitMan : Pacman(parent) {
-        override suspend fun createModuleScope(name: String): ModuleScope? = modules[name]?.deferredModule?.await()
-    }
-
-    private val readyModules by lazy {
-        globalDefer {
-            modules.entries.map { it.key to it.value.deferredModule.await() }.toMap()
-        }
-    }
-
-
-    override suspend fun createModuleScope(name: String): ModuleScope? =
-        readyModules.await()[name]
-}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt

@@ -1,33 +1,47 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.pacman.ImportProvider
+
 /**
- * Module scope supports importing and contains the [pacman]; it should be the same
+ * Module scope supports importing and contains the [importProvider]; it should be the same
  * used in [Compiler];
  */
 class ModuleScope(
-    val pacman: Pacman,
+    val importProvider: ImportProvider,
     pos: Pos = Pos.builtIn,
-    val packageName: String
+    override val packageName: String
-) : Scope(pacman.rootScope, Arguments.EMPTY, pos) {
+) : Scope(importProvider.rootScope, Arguments.EMPTY, pos) {
 
-    constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName)
+    constructor(importProvider: ImportProvider, source: Source) : this(importProvider, source.startPos, source.fileName)
-
-    override suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>?) {
-        pacman.prepareImport(pos, name, symbols)
-    }
 
     /**
-     * Import symbols into the scope. It _is called_ after the module is imported by [checkImport].
+     * Import symbols into the scope. It _is called_ after the module is imported by [ImportProvider.prepareImport]
-     * If [checkImport] was not called, the symbols will not be imported with exception as module is not found.
+     * which checks symbol availability and accessibility prior to execution.
+     * @param scope where to copy symbols from this module
+     * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
      */
-    override suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>?) {
+    override suspend fun importInto(scope: Scope, symbols: Map<String, String>?) {
-        pacman.performImport(scope, name, symbols)
+        val symbolsToImport = symbols?.keys?.toMutableSet()
+        for ((symbol, record) in this.objects) {
+            if (record.visibility.isPublic) {
+                val newName = symbols?.let { ss: Map<String, String> ->
+                    ss[symbol]
+                        ?.also { symbolsToImport!!.remove(it) }
+                        ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
+                } ?: symbol
+                if (newName in scope.objects)
+                    scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
+                scope.objects[newName] = record
+            }
+        }
+        if (!symbolsToImport.isNullOrEmpty())
+            scope.raiseSymbolNotFound("symbols $packageName.{$symbolsToImport} are.were not found")
     }
 
-    val packageNameObj by lazy { ObjString(packageName).asReadonly}
+    val packageNameObj by lazy { ObjString(packageName).asReadonly }
 
     override fun get(name: String): ObjRecord? {
-        return if( name == "__PACKAGE__")
+        return if (name == "__PACKAGE__")
             packageNameObj
         else
             super.get(name)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt

@@ -1,69 +0,0 @@
-package net.sergeych.lyng
-
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
-
-/**
- * Package manager. Chained manager, too simple. Override [createModuleScope] to return either
- * valid [ModuleScope] or call [parent] - or return null.
- */
-abstract class Pacman(
-    val parent: Pacman? = null,
-    val rootScope: Scope = parent!!.rootScope,
-    val securityManager: SecurityManager = parent!!.securityManager
-) {
-    private val opScopes = Mutex()
-
-    private val cachedScopes = mutableMapOf<String, Scope>()
-
-    /**
-     * Create a new module scope if this pacman can import the module, return null otherwise so
-     * the manager can decide what to do
-     */
-    abstract suspend fun createModuleScope(name: String): ModuleScope?
-
-    suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?) {
-        if (!securityManager.canImportModule(name))
-            throw ImportException(pos, "Module $name is not allowed")
-        symbols?.keys?.forEach {
-            if (!securityManager.canImportSymbol(name, it)) throw ImportException(
-                pos,
-                "Symbol $name.$it is not allowed"
-            )
-        }
-        // if we can import the module, cache it, or go further
-        opScopes.withLock {
-            cachedScopes[name] ?: createModuleScope(name)?.let { cachedScopes[name] = it }
-        } ?: parent?.prepareImport(pos, name, symbols) ?: throw ImportException(pos, "Module $name is not found")
-    }
-
-    suspend fun performImport(scope: Scope, name: String, symbols: Map<String, String>?) {
-        val module = opScopes.withLock { cachedScopes[name] }
-            ?: scope.raiseSymbolNotFound("module $name not found")
-        val symbolsToImport = symbols?.keys?.toMutableSet()
-        for ((symbol, record) in module.objects) {
-            if (record.visibility.isPublic) {
-                val newName = symbols?.let { ss: Map<String, String> ->
-                    ss[symbol]
-                        ?.also { symbolsToImport!!.remove(it) }
-                        ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
-                } ?: symbol
-                if (newName in scope.objects)
-                    scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
-                scope.objects[newName] = record
-            }
-        }
-        if (!symbolsToImport.isNullOrEmpty())
-            scope.raiseSymbolNotFound("symbols $name.{$symbolsToImport} are.were not found")
-    }
-
-    companion object {
-        val emptyAllowAll = object : Pacman(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
-            override suspend fun createModuleScope(name: String): ModuleScope? {
-                return null
-            }
-
-        }
-    }
-
-}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.pacman.ImportProvider
+
 /**
  * Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes.
  * Each block usually creates a scope. Accessing Lyng closures usually is done via a scope.
@@ -16,6 +18,8 @@ open class Scope(
     var thisObj: Obj = ObjVoid,
     var skipScopeCreation: Boolean = false,
 ) {
+    open val packageName: String = "<anonymous package>"
+
     constructor(
         args: Arguments = Arguments.EMPTY,
         pos: Pos = Pos.builtIn,
@@ -137,16 +141,17 @@ open class Scope(
     suspend fun eval(source: Source): Obj =
         Compiler.compile(
             source,
-            (this as? ModuleScope)?.pacman ?: Pacman.emptyAllowAll
+            (this as? ModuleScope)?.importProvider ?: ImportProvider.emptyAllowAll
-            ).execute(this)
+        ).execute(this)
 
     fun containsLocal(name: String): Boolean = name in objects
 
-    open suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>? = null) {
+    /**
-        throw ImportException(pos, "Import is not allowed here: $name")
+     * Some scopes can be imported into other scopes, like [ModuleScope]. Those must correctly implement this method.
-    }
+     * @param scope where to copy symbols from this module
-
+     * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
-    open suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>? = null) {
+     */
-        scope.raiseError(ObjIllegalOperationException(scope,"Import is not allowed here: import $name"))
+    open suspend fun importInto(scope: Scope, symbols: Map<String, String>? = null) {
+        scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName"))
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt

@@ -0,0 +1,48 @@
+package net.sergeych.lyng.pacman
+
+import net.sergeych.lyng.*
+
+/**
+ * Package manager INTERFACE (abstract class). Performs import routines
+ * using abstract [createModuleScope] method ot be implemented by heirs.
+ *
+ * Notice that [createModuleScope] is responsible for caching the modules;
+ * base class relies on caching. This is not implemented here as the correct
+ * caching strategy depends on the import provider
+ */
+abstract class ImportProvider(
+    val rootScope: Scope = Script.defaultScope,
+    val securityManager: SecurityManager = SecurityManager.allowAll
+) {
+    /**
+     * Find an import and create a scope for it. This method must implement caching so repeated
+     * imports are not repeatedly loaded and parsed and should be cheap.
+     *
+     * @throws ImportException if the module is not found
+     */
+    abstract suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope
+
+    /**
+     * Check that the import is possible and allowed. This method is called on compile time by [Compiler];
+     * actual module loading is performed by [ModuleScope.importInto]
+     */
+    suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?): ModuleScope {
+        if (!securityManager.canImportModule(name))
+            throw ImportException(pos, "Module $name is not allowed")
+        symbols?.keys?.forEach {
+            if (!securityManager.canImportSymbol(name, it)) throw ImportException(
+                pos,
+                "Symbol $name.$it is not allowed"
+            )
+        }
+        return createModuleScope(pos, name)
+    }
+
+    companion object {
+        val emptyAllowAll = object : ImportProvider(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
+            override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
+                throw ImportException(pos, "Empty import provider can't be used directly")
+            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt

@@ -0,0 +1,63 @@
+package net.sergeych.lyng.pacman
+
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import net.sergeych.lyng.*
+
+/**
+ * The import provider that imports sources available in memory.
+ */
+class InlineSourcesImportProvider(val sources: List<Source>,
+                                  rootScope: ModuleScope = ModuleScope(emptyAllowAll, Source.builtIn),
+                                  securityManager: SecurityManager = SecurityManager.allowAll
+) : ImportProvider(rootScope, securityManager) {
+
+    private class Entry(
+        val source: Source,
+        var scope: ModuleScope? = null,
+    )
+
+    private val inner = InitMan()
+
+    private val modules = run {
+        val result = mutableMapOf<String, Entry>()
+        for (source in sources) {
+            val name = source.extractPackageName()
+            result[name] = Entry(source)
+        }
+        result
+    }
+
+    private var access = Mutex()
+
+    /**
+     * Inner provider does not lock [access], the only difference; it is meant to be used
+     * exclusively by the coroutine that starts actual import chain
+     */
+    private inner class InitMan : ImportProvider() {
+        override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
+            return doImport(packageName, pos)
+        }
+    }
+
+    /**
+     * External interface, thread-safe. Can suspend until actual import is done. implements caching.
+     */
+    override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope =
+        access.withLock {
+            doImport(packageName, pos)
+        }
+
+    /**
+     * Perform actual import or return ready scope. __It must only be called when
+     * [access] is locked__, e.g. only internally
+     */
+    private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
+        modules[packageName]?.scope?.let { return it }
+        val entry = modules[packageName] ?: throw ImportException(pos, "Unknown package $packageName")
+        return ModuleScope(inner, pos, packageName).apply {
+            entry.scope = this
+            eval(entry.source)
+        }
+    }
+}

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -1,8 +1,8 @@
-
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.test.runTest
 import net.sergeych.lyng.*
+import net.sergeych.lyng.pacman.InlineSourcesImportProvider
 import kotlin.test.*
 
 class ScriptTest {
@@ -2250,18 +2250,21 @@ class ScriptTest {
 
     @Test
     fun testLet() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x=0,y=0)
             assert( Point() is Object) 
             Point().let { println(it.x, it.y) }
             val x = null
             x?.let { println(it.x, it.y) }
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testApply() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x,y)
             // see the difference: apply changes this to newly created Point:
             val p = Point(1,2).apply { 
@@ -2270,12 +2273,14 @@ class ScriptTest {
             assertEquals(p, Point(2,3))
             >>> void
 
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testApplyThis() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x,y)
             // see the difference: apply changes this to newly created Point:
             val p = Point(1,2).apply { 
@@ -2284,12 +2289,14 @@ class ScriptTest {
             assertEquals(p, Point(2,3))
             >>> void
 
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testExtend() = runTest() {
-        eval("""
+        eval(
+            """
             
             fun Int.isEven() {
                 this % 2 == 0
@@ -2311,7 +2318,8 @@ class ScriptTest {
             assert( 12.1.isInteger() == false )
             assert( "5".isInteger() )
             assert( ! "5.2".isInteger() )
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
@@ -2319,18 +2327,20 @@ class ScriptTest {
         val c = Scope()
         val arr = c.eval("[1,2,3]")
         // array is iterable so we can:
-        assertEquals(listOf(1,2,3),  arr.toFlow(c).map { it.toInt() }.toList())
+        assertEquals(listOf(1, 2, 3), arr.toFlow(c).map { it.toInt() }.toList())
     }
 
     @Test
     fun testAssociateBy() = runTest() {
-        eval("""
+        eval(
+            """
             val m = [123, 456].associateBy { "k:%s"(it) }
             println(m)
             assertEquals(123, m["k:123"])
             assertEquals(456, m["k:456"])
-            """)
+            """
-        listOf(1,2,3).associateBy { it * 10 }
+        )
+        listOf(1, 2, 3).associateBy { it * 10 }
     }
 
 //    @Test
@@ -2354,7 +2364,7 @@ class ScriptTest {
             
             fun foo() { "foo1" }
             """.trimIndent()
-        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
+        val pm = InlineSourcesImportProvider(listOf(Source("foosrc", foosrc)))
 
         val src = """
             import lyng.foo
@@ -2380,11 +2390,12 @@ class ScriptTest {
             
             fun bar() { "bar1" }
             """.trimIndent()
-        val pm = InlineSourcesPacman(
+        val pm = InlineSourcesImportProvider(
-            Pacman.emptyAllowAll, listOf(
+            listOf(
                 Source("barsrc", barsrc),
                 Source("foosrc", foosrc),
-            ))
+            )
+        )
 
         val src = """
             import lyng.foo
@@ -2396,4 +2407,37 @@ class ScriptTest {
         assertEquals("foo1 / bar1", scope.eval(src).toString())
     }
 
+    @Test
+    fun testImportsCircular() = runTest {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            import lyng.foo
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesImportProvider(
+            listOf(
+                Source("barsrc", barsrc),
+                Source("foosrc", foosrc),
+            )
+        )
+
+        val src = """
+            import lyng.bar
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
 }

lynglib/src/jvmTest/kotlin/OtherTests.kt

@@ -1,6 +1,9 @@
 import junit.framework.TestCase.assertEquals
 import kotlinx.coroutines.runBlocking
-import net.sergeych.lyng.*
+import net.sergeych.lyng.ModuleScope
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.pacman.InlineSourcesImportProvider
+import net.sergeych.lyng.toSource
 import kotlin.test.Test
 
 class OtherTests {
@@ -18,8 +21,8 @@ class OtherTests {
             
             fun bar() { "bar1" }
             """.trimIndent()
-        val pm = InlineSourcesPacman(
+        val pm = InlineSourcesImportProvider(
-            Pacman.emptyAllowAll, listOf(
+            listOf(
             Source("foosrc", foosrc),
             Source("barsrc", barsrc),
         ))