fix #38 ImportManager integrated into Scope tree and all systems

docs/samples/helloworld.lyng

@@ -1,3 +1,3 @@
 #!/bin/env jlyng
 
-println("Hello, world2! "+ARGV);
+println("Hello, world!");

lynglib/build.gradle.kts

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 group = "net.sergeych"
-version = "0.7.1-SNAPSHOT"
+version = "0.7.2-SNAPSHOT"
 
 buildscript {
     repositories {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt

@@ -54,8 +54,8 @@ data class ArgsDeclaration(val params: List<Item>, val endTokenType: Token.Type)
                     i < callArgs.size -> callArgs[i]
                     a.defaultValue != null -> a.defaultValue.execute(scope)
                     else -> {
-                        println("callArgs: ${callArgs.joinToString()}")
+//                        println("callArgs: ${callArgs.joinToString()}")
-                        println("tailBlockMode: ${arguments.tailBlockMode}")
+//                        println("tailBlockMode: ${arguments.tailBlockMode}")
                         scope.raiseIllegalArgument("too few arguments for the call")
                     }
                 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -7,7 +7,7 @@ import net.sergeych.lyng.pacman.ImportProvider
  */
 class Compiler(
     val cc: CompilerContext,
-    val importProvider: ImportProvider = ImportProvider.emptyAllowAll,
+    val importManager: ImportProvider = Script.defaultImportManager,
     @Suppress("UNUSED_PARAMETER")
     settings: Settings = Settings()
 ) {
@@ -43,7 +43,7 @@ class Compiler(
                         cc.next()
                         val pos = cc.currentPos()
                         val name = loadQualifiedName()
-                        val module = importProvider.prepareImport(pos, name, null)
+                        val module = importManager.prepareImport(pos, name, null)
                         statements += statement {
                             module.importInto(this, null)
                             ObjVoid
@@ -1620,7 +1620,7 @@ class Compiler(
 
     companion object {
 
-        suspend fun compile(source: Source, importProvider: ImportProvider = ImportProvider.emptyAllowAll): Script {
+        suspend fun compile(source: Source, importProvider: ImportProvider = Script.defaultImportManager): Script {
             return Compiler(CompilerContext(parseLyng(source)),importProvider).parseScript()
         }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt

@@ -109,7 +109,7 @@ private class Parser(fromPos: Pos) {
             '/' -> when (currentChar) {
                 '/' -> {
                     pos.advance()
-                    Token(loadToEnd().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
+                    Token(loadToEndOfLine().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
                 }
 
                 '*' -> {
@@ -445,7 +445,7 @@ private class Parser(fromPos: Pos) {
         }
     }
 
-    private fun loadToEnd(): String {
+    private fun loadToEndOfLine(): String {
         val result = StringBuilder()
         val l = pos.line
         do {
@@ -479,4 +479,10 @@ private class Parser(fromPos: Pos) {
         return null
     }
 
+    init {
+        // skip shebang
+        if( pos.readFragment("#!") )
+            loadToEndOfLine()
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -1,15 +1,20 @@
 package net.sergeych.lyng
 
-import net.sergeych.lyng.pacman.ImportProvider
+import net.sergeych.lyng.pacman.ImportManager
 
 /**
  * Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes.
  * Each block usually creates a scope. Accessing Lyng closures usually is done via a scope.
  *
- * There are special types of scopes:
+ * To create default scope, use default `Scope()` constructor, it will create a scope with a parent
+ * module scope with default [ImportManager], you can access with [importManager] as needed.
  *
- * - [Script.defaultScope] - root scope for a script, safe one
+ * If you want to create [ModuleScope] by hand, try [importManager] and [ImportManager.newModule],
- * - [AppliedScope] - scope used to apply a closure to some thisObj scope
+ * or [ImportManager.newModuleAt].
+ *
+ *  There are special types of scopes:
+ *
+ *  - [AppliedScope] - scope used to apply a closure to some thisObj scope
  */
 open class Scope(
     val parent: Scope?,
@@ -24,7 +29,7 @@ open class Scope(
         args: Arguments = Arguments.EMPTY,
         pos: Pos = Pos.builtIn,
     )
-            : this(Script.defaultScope, args, pos)
+            : this(Script.defaultImportManager.newModuleAt(pos), args, pos)
 
     fun raiseNotImplemented(what: String = "operation"): Nothing = raiseError("$what is not implemented")
 
@@ -141,7 +146,7 @@ open class Scope(
     suspend fun eval(source: Source): Obj =
         Compiler.compile(
             source,
-            (this as? ModuleScope)?.importProvider ?: ImportProvider.emptyAllowAll
+            (this as? ModuleScope)?.importProvider ?: Script.defaultImportManager
         ).execute(this)
 
     fun containsLocal(name: String): Boolean = name in objects
@@ -154,4 +159,19 @@ open class Scope(
     open suspend fun importInto(scope: Scope, symbols: Map<String, String>? = null) {
         scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName"))
     }
+
+    /**
+     * Find a first [ImportManager] in this Scope hierarchy. Normally there should be one. Found instance is cached.
+     *
+     * Use it to register your package sources, see [ImportManager] features.
+     *
+     * @throws IllegalStateException if there is no such manager (if you create some specific scope with no manager,
+     *      then you knew what you did)
+     */
+    val importManager: ImportManager by lazy {
+        if( this is ModuleScope )
+            (importProvider as? ImportManager)?.let { return@lazy it }
+        parent?.importManager ?: throw IllegalStateException("this scope has no manager in the chain")
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt

@@ -1,6 +1,7 @@
 package net.sergeych.lyng
 
 import kotlinx.coroutines.delay
+import net.sergeych.lyng.pacman.ImportManager
 import kotlin.math.*
 
 class Script(
@@ -17,10 +18,11 @@ class Script(
         return lastResult
     }
 
-    suspend fun execute() = execute(defaultScope.copy(pos = pos))
+    suspend fun execute() = execute(defaultImportManager.newModule())
 
     companion object {
-        val defaultScope: Scope = Scope().apply {
+
+        private val rootScope: Scope = Scope(null).apply {
             ObjException.addExceptionsToContext(this)
             addFn("println") {
                 for ((i, a) in args.withIndex()) {
@@ -170,8 +172,9 @@ class Script(
             getOrCreateNamespace("Math").apply {
                 addConst("PI", pi)
             }
-
-
         }
+
+        val defaultImportManager: ImportManager by lazy { ImportManager(rootScope, SecurityManager.allowAll) }
+
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt

@@ -0,0 +1,126 @@
+package net.sergeych.lyng.pacman
+
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import net.sergeych.lyng.*
+
+/**
+ * Import manager allow to register packages with builder lambdas and act as an
+ * [ImportProvider]. Note that packages _must be registered_ first with [addPackage],
+ * [addSourcePackages] or [addTextPackages]. Registration is cheap, actual package
+ * building is lazily performed on [createModuleScope], when the package will
+ * be first imported.
+ *
+ * It is possible to register new packages at any time, but it is not allowed to override
+ * packages already registered.
+ */
+class ImportManager(
+    rootScope: Scope = Script.defaultImportManager.newModule(),
+    securityManager: SecurityManager = SecurityManager.allowAll
+): ImportProvider(rootScope, securityManager) {
+
+    private inner class Entry(
+        val packageName: String,
+        val builder: suspend (ModuleScope) -> Unit,
+        var cachedScope: ModuleScope? = null
+    ) {
+
+        suspend fun getScope(pos: Pos): ModuleScope {
+            cachedScope?.let { return it }
+            return ModuleScope(inner, pos, packageName).apply {
+                cachedScope = this
+                builder(this)
+            }
+        }
+    }
+
+
+    /**
+     * Inner provider does not lock [access], the only difference; it is meant to be used
+     * exclusively by the coroutine that starts actual import chain
+     */
+    private inner class InternalProvider : ImportProvider(rootScope) {
+        override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope {
+            return doImport(packageName, pos)
+        }
+    }
+
+    /**
+     * Inner module import provider used to prepare lazily prepared modules
+     */
+    private val inner = InternalProvider()
+
+
+    private val imports = mutableMapOf<String, Entry>()
+    private val access = Mutex()
+
+    /**
+     * Register new package that can be imported. It is not possible to unregister or
+     * update package already registered.
+     *
+     * Packages are lazily created when first imported somewhere, so the registration is
+     * cheap; the recommended procedure is to register all available packages prior to
+     * compile with this.
+     *
+     * @param name package name
+     * @param builder lambda to create actual package using the given [ModuleScope]
+     */
+    suspend fun addPackage(name: String, builder: suspend (ModuleScope) -> Unit) {
+        access.withLock {
+            if (name in imports)
+                throw IllegalArgumentException("Package $name already exists")
+            imports[name] = Entry(name, builder)
+        }
+    }
+
+    /**
+     * Bulk [addPackage] with slightly better performance
+     */
+    @Suppress("unused")
+    suspend fun addPackages(registrationData: List<Pair<String, suspend (ModuleScope) -> Unit>>) {
+        access.withLock {
+            for (pp in registrationData) {
+                if (pp.first in imports)
+                    throw IllegalArgumentException("Package ${pp.first} already exists")
+                imports[pp.first] = Entry(pp.first, pp.second)
+            }
+        }
+    }
+
+    /**
+     * Perform actual import or return ready scope. __It must only be called when
+     * [access] is locked__, e.g. only internally
+     */
+    private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
+        val entry = imports[packageName] ?: throw ImportException(pos, "package not found: $packageName")
+        return entry.getScope(pos)
+    }
+
+    override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope =
+        doImport(packageName, pos)
+
+    /**
+     * Add packages that only need to compile [Source].
+     */
+    suspend fun addSourcePackages(vararg sources: Source) {
+        for( s in sources) {
+            addPackage(s.extractPackageName()) {
+                it.eval(s)
+            }
+
+        }
+    }
+
+    /**
+     * Add source packages using package name as [Source.fileName], for simplicity
+     */
+    suspend fun addTextPackages(vararg sourceTexts: String) {
+        for( s in sourceTexts) {
+            var source = Source("tmp", s)
+            val packageName = source.extractPackageName()
+            source = Source(packageName, s)
+            addPackage(packageName) { it.eval(source)}
+        }
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt

@@ -11,7 +11,7 @@ import net.sergeych.lyng.*
  * caching strategy depends on the import provider
  */
 abstract class ImportProvider(
-    val rootScope: Scope = Script.defaultScope,
+    val rootScope: Scope,
     val securityManager: SecurityManager = SecurityManager.allowAll
 ) {
     /**
@@ -38,11 +38,11 @@ abstract class ImportProvider(
         return createModuleScope(pos, name)
     }
 
-    companion object {
+    fun newModule() = newModuleAt(Pos.builtIn)
-        val emptyAllowAll = object : ImportProvider(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
+
-            override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
+    fun newModuleAt(pos: Pos): ModuleScope =
-                throw ImportException(pos, "Empty import provider can't be used directly")
+        ModuleScope(this, pos, "unknown")
-            }
+}
-        }
+
-    }
+
-}
+

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt

@@ -1,63 +1,35 @@
 package net.sergeych.lyng.pacman
 
-import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.CompletableDeferred
-import kotlinx.coroutines.sync.withLock
 import net.sergeych.lyng.*
+import net.sergeych.mp_tools.globalLaunch
 
 /**
- * The import provider that imports sources available in memory.
+ * The sample import provider that imports sources available in memory.
+ * on construction time.
+ *
+ * Actually it is left here only as a demo.
  */
-class InlineSourcesImportProvider(val sources: List<Source>,
+class InlineSourcesImportProvider(sources: List<Source>,
-                                  rootScope: ModuleScope = ModuleScope(emptyAllowAll, Source.builtIn),
+                                  rootScope: ModuleScope = Script.defaultImportManager.newModule(),
                                   securityManager: SecurityManager = SecurityManager.allowAll
-) : ImportProvider(rootScope, securityManager) {
+) : ImportProvider(rootScope) {
 
-    private class Entry(
+    private val manager = ImportManager(rootScope, securityManager)
-        val source: Source,
-        var scope: ModuleScope? = null,
-    )
 
-    private val inner = InitMan()
+    private val readyManager = CompletableDeferred<ImportManager>()
 
-    private val modules = run {
+    /**
-        val result = mutableMapOf<String, Entry>()
+     * This implementation only
-        for (source in sources) {
+     */
-            val name = source.extractPackageName()
+    override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope {
-            result[name] = Entry(source)
+        return readyManager.await().createModuleScope(pos, packageName)
-        }
-        result
     }
 
-    private var access = Mutex()
+    init {
-
+        globalLaunch {
-    /**
+            manager.addSourcePackages(*sources.toTypedArray())
-     * Inner provider does not lock [access], the only difference; it is meant to be used
+            readyManager.complete(manager)
-     * exclusively by the coroutine that starts actual import chain
-     */
-    private inner class InitMan : ImportProvider() {
-        override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope {
-            return doImport(packageName, pos)
-        }
-    }
-
-    /**
-     * External interface, thread-safe. Can suspend until actual import is done. implements caching.
-     */
-    override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope =
-        access.withLock {
-            doImport(packageName, pos)
-        }
-
-    /**
-     * Perform actual import or return ready scope. __It must only be called when
-     * [access] is locked__, e.g. only internally
-     */
-    private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
-        modules[packageName]?.scope?.let { return it }
-        val entry = modules[packageName] ?: throw ImportException(pos, "Unknown package $packageName")
-        return ModuleScope(inner, pos, packageName).apply {
-            entry.scope = this
-            eval(entry.source)
         }
     }
 }

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -604,7 +604,7 @@ class ScriptTest {
     }
 
     @Test
-    fun testAssignArgumentsmiddleEllipsis() = runTest {
+    fun testAssignArgumentsMiddleEllipsis() = runTest {
         val ttEnd = Token.Type.RBRACE
         val pa = ArgsDeclaration(
             listOf(
@@ -2440,4 +2440,24 @@ class ScriptTest {
         assertEquals("foo1 / bar1", scope.eval(src).toString())
     }
 
+    @Test
+    fun testDefaultImportManager() = runTest {
+        val scope = Scope()
+        assertFails {
+            scope.eval("""
+                import foo
+                foo()
+                """.trimIndent())
+        }
+        scope.importManager.addTextPackages("""
+            package foo
+            
+            fun foo() { "bar" }
+        """.trimIndent())
+        scope.eval("""
+                import foo
+                assertEquals( "bar", foo())
+                """.trimIndent())
+    }
+
 }