fixed import inside sources in InlineSourcesPacman

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -20,6 +20,10 @@ class Compiler(
         // package level declarations
         do {
             val t = cc.current()
+            if(t.type == Token.Type.NEWLINE || t.type == Token.Type.SINLGE_LINE_COMMENT || t.type == Token.Type.MULTILINE_COMMENT) {
+                cc.next()
+                continue
+            }
             if (t.type == Token.Type.ID) {
                 when (t.value) {
                     "package" -> {
@@ -42,6 +46,7 @@ class Compiler(
                             pacman.performImport(this,name,null)
                             ObjVoid
                         }
+                        continue
                     }
                 }
             }
@@ -64,6 +69,7 @@ class Compiler(
                 t = cc.next()
             }
         }
+        cc.previous()
         return result.toString()
     }
 
@@ -1616,13 +1622,6 @@ class Compiler(
             return Compiler(CompilerContext(parseLyng(source)),pacman).parseScript()
         }
 
-        suspend fun compilePackage(source: Source): Pair<String, Script> {
-            val c = Compiler(CompilerContext(parseLyng(source)))
-            val script = c.parseScript()
-            if (c.packageName == null) throw ScriptError(source.startPos, "package not set")
-            return c.packageName!! to script
-        }
-
         private var lastPriority = 0
         val allOps = listOf(
             // assignments, lowest priority

lynglib/src/commonMain/kotlin/net/sergeych/lyng/CompilerContext.kt

@@ -164,7 +164,10 @@ class CompilerContext(val tokens: List<Token>) {
      * Note that [Token.Type.EOF] is not considered a whitespace token.
      */
     fun skipWsTokens(): Token {
-        while( current().type in wstokens ) next()
+        while( current().type in wstokens ) {
+            println("skipws ${current()}")
+            next()
+        }
         return next()
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt

@@ -0,0 +1,48 @@
+package net.sergeych.lyng
+
+import kotlinx.coroutines.CompletableDeferred
+import net.sergeych.mp_tools.globalDefer
+import net.sergeych.mp_tools.globalLaunch
+
+/**
+ * Naive source-based pacman that compiles all sources first, before first import could be resolved.
+ * It supports imports between [sources] but does not resolve nor detect cyclic imports which
+ * are not supported.
+ */
+class InlineSourcesPacman(pacman: Pacman, val sources: List<Source>) : Pacman(pacman) {
+
+    data class Entry(val source: Source, val deferredModule: CompletableDeferred<ModuleScope> = CompletableDeferred())
+
+
+    private val modules = run {
+        val result = mutableMapOf<String, Entry>()
+        for (source in sources) {
+            val name = source.extractPackageName()
+            result[name] = Entry(source)
+        }
+        val inner = InitMan()
+
+        for ((name, entry) in result) {
+            globalLaunch {
+                val scope = ModuleScope(inner, entry.source.startPos, name)
+                Compiler.compile(entry.source, inner).execute(scope)
+                entry.deferredModule.complete(scope)
+            }
+        }
+        result
+    }
+
+    inner class InitMan : Pacman(parent) {
+        override suspend fun createModuleScope(name: String): ModuleScope? = modules[name]?.deferredModule?.await()
+    }
+
+    private val readyModules by lazy {
+        globalDefer {
+            modules.entries.map { it.key to it.value.deferredModule.await() }.toMap()
+        }
+    }
+
+
+    override suspend fun createModuleScope(name: String): ModuleScope? =
+        readyModules.await()[name]
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt

@@ -0,0 +1,37 @@
+package net.sergeych.lyng
+
+/**
+ * Module scope supports importing and contains the [pacman]; it should be the same
+ * used in [Compiler];
+ */
+class ModuleScope(
+    val pacman: Pacman,
+    pos: Pos = Pos.builtIn,
+    val packageName: String
+) : Scope(pacman.rootScope, Arguments.EMPTY, pos) {
+
+    constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName)
+
+    override suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>?) {
+        pacman.prepareImport(pos, name, symbols)
+    }
+
+    /**
+     * Import symbols into the scope. It _is called_ after the module is imported by [checkImport].
+     * If [checkImport] was not called, the symbols will not be imported with exception as module is not found.
+     */
+    override suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>?) {
+        pacman.performImport(scope, name, symbols)
+    }
+
+    val packageNameObj by lazy { ObjString(packageName).asReadonly}
+
+    override fun get(name: String): ObjRecord? {
+        return if( name == "__PACKAGE__")
+            packageNameObj
+        else
+            super.get(name)
+    }
+}
+
+

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt

@@ -1,9 +1,7 @@
 package net.sergeych.lyng
 
-import kotlinx.coroutines.Deferred
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
-import net.sergeych.mp_tools.globalDefer
 
 /**
  * Package manager. Chained manager, too simple. Override [createModuleScope] to return either
@@ -45,13 +43,11 @@ abstract class Pacman(
         val symbolsToImport = symbols?.keys?.toMutableSet()
         for ((symbol, record) in module.objects) {
             if (record.visibility.isPublic) {
-                println("import $name: $symbol: $record")
                 val newName = symbols?.let { ss: Map<String, String> ->
                     ss[symbol]
                         ?.also { symbolsToImport!!.remove(it) }
                         ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
                 } ?: symbol
-                println("import $name.$symbol as $newName")
                 if (newName in scope.objects)
                     scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
                 scope.objects[newName] = record
@@ -70,61 +66,4 @@ abstract class Pacman(
         }
     }
 
-}
+}
-
-/**
- * Module scope supports importing and contains the [pacman]; it should be the same
- * used in [Compiler];
- */
-class ModuleScope(
-    val pacman: Pacman,
-    pos: Pos = Pos.builtIn,
-    val packageName: String
-) : Scope(pacman.rootScope, Arguments.EMPTY, pos) {
-
-    constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName)
-
-    override suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>?) {
-        pacman.prepareImport(pos, name, symbols)
-    }
-
-    /**
-     * Import symbols into the scope. It _is called_ after the module is imported by [checkImport].
-     * If [checkImport] was not called, the symbols will not be imported with exception as module is not found.
-     */
-    override suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>?) {
-        pacman.performImport(scope, name, symbols)
-    }
-
-    val packageNameObj by lazy { ObjString(packageName).asReadonly}
-
-    override fun get(name: String): ObjRecord? {
-        return if( name == "__PACKAGE__")
-            packageNameObj
-        else
-            super.get(name)
-    }
-}
-
-
-class InlineSourcesPacman(pacman: Pacman,val sources: List<Source>) : Pacman(pacman) {
-
-    val modules: Deferred<Map<String,Deferred<ModuleScope>>> = globalDefer {
-        val result = mutableMapOf<String, Deferred<ModuleScope>>()
-        for (source in sources) {
-            // retrieve the module name and script for deferred execution:
-            val (name, script) = Compiler.compilePackage(source)
-            // scope is created used pacman's root scope:
-            val scope = ModuleScope(this@InlineSourcesPacman, source.startPos, name)
-            // we execute scripts in parallel which allow cross-imports to some extent:
-            result[name] = globalDefer { script.execute(scope); scope }
-        }
-        result
-    }
-
-    override suspend fun createModuleScope(name: String): ModuleScope? =
-        modules.await()[name]?.await()
-
-}
-
-

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -137,7 +137,7 @@ open class Scope(
     suspend fun eval(source: Source): Obj =
         Compiler.compile(
             source,
-            (this as? ModuleScope)?.pacman?.also { println("pacman found: $pacman")} ?: Pacman.emptyAllowAll
+            (this as? ModuleScope)?.pacman ?: Pacman.emptyAllowAll
             ).execute(this)
 
     fun containsLocal(name: String): Boolean = name in objects

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Source.kt

@@ -12,4 +12,15 @@ class Source(val fileName: String, text: String) {
     val startPos: Pos = Pos(this, 0, 0)
 
     fun posAt(line: Int, column: Int): Pos = Pos(this, line, column)
+
+    fun extractPackageName(): String {
+        for ((n,line) in lines.withIndex()) {
+            if( line.isBlank() || line.isEmpty() )
+                continue
+            if( line.startsWith("package ") )
+                return line.substring(8).trim()
+            else throw ScriptError(Pos(this, n, 0),"package declaration expected")
+        }
+        throw ScriptError(Pos(this, 0, 0),"package declaration expected")
+    }
 }

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -2333,19 +2333,19 @@ class ScriptTest {
         listOf(1,2,3).associateBy { it * 10 }
     }
 
-    @Test
+//    @Test
-    fun testImports1() = runTest() {
+//    fun testImports1() = runTest() {
-        val foosrc = """
+//        val foosrc = """
-            package lyng.foo
+//            package lyng.foo
-            
+//
-            fun foo() { "foo1" }
+//            fun foo() { "foo1" }
-            """.trimIndent()
+//            """.trimIndent()
-        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
+//        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
-        assertNotNull(pm.modules.await()["lyng.foo"])
+//        assertNotNull(pm.modules["lyng.foo"])
-        assertIs<ModuleScope>(pm.modules.await()["lyng.foo"]!!.await())
+//        assertIs<ModuleScope>(pm.modules["lyng.foo"]!!.deferredModule.await())
 
-        assertEquals("foo1", pm.modules.await()["lyng.foo"]!!.await().eval("foo()").toString())
+//        assertEquals("foo1", pm.modules["lyng.foo"]!!.deferredModule.await().eval("foo()").toString())
-    }
+//    }
 
     @Test
     fun testImports2() = runTest() {
@@ -2365,4 +2365,35 @@ class ScriptTest {
         val scope = ModuleScope(pm, src)
         assertEquals("foo1", scope.eval(src).toString())
     }
+
+    @Test
+    fun testImports3() = runTest {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesPacman(
+            Pacman.emptyAllowAll, listOf(
+                Source("barsrc", barsrc),
+                Source("foosrc", foosrc),
+            ))
+
+        val src = """
+            import lyng.foo
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
 }

lynglib/src/jvmTest/kotlin/OtherTests.kt

@@ -0,0 +1,37 @@
+import junit.framework.TestCase.assertEquals
+import kotlinx.coroutines.runBlocking
+import net.sergeych.lyng.*
+import kotlin.test.Test
+
+class OtherTests {
+    @Test
+    fun testImports3() = runBlocking {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesPacman(
+            Pacman.emptyAllowAll, listOf(
+            Source("foosrc", foosrc),
+            Source("barsrc", barsrc),
+        ))
+
+        val src = """
+            import lyng.foo
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
+}