diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt index fded983..951b194 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt @@ -1,11 +1,13 @@ package net.sergeych.lyng +import net.sergeych.lyng.pacman.ImportProvider + /** * The LYNG compiler. */ class Compiler( val cc: CompilerContext, - val pacman: Pacman = Pacman.emptyAllowAll, + val importProvider: ImportProvider = ImportProvider.emptyAllowAll, @Suppress("UNUSED_PARAMETER") settings: Settings = Settings() ) { @@ -41,9 +43,9 @@ class Compiler( cc.next() val pos = cc.currentPos() val name = loadQualifiedName() - pacman.prepareImport(pos, name, null) + val module = importProvider.prepareImport(pos, name, null) statements += statement { - pacman.performImport(this,name,null) + module.importInto(this, null) ObjVoid } continue @@ -1618,8 +1620,8 @@ class Compiler( companion object { - suspend fun compile(source: Source,pacman: Pacman = Pacman.emptyAllowAll): Script { - return Compiler(CompilerContext(parseLyng(source)),pacman).parseScript() + suspend fun compile(source: Source, importProvider: ImportProvider = ImportProvider.emptyAllowAll): Script { + return Compiler(CompilerContext(parseLyng(source)),importProvider).parseScript() } private var lastPriority = 0 diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt deleted file mode 100644 index 95cb6c1..0000000 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/InlineSourcesPacman.kt +++ /dev/null @@ -1,48 +0,0 @@ -package net.sergeych.lyng - -import kotlinx.coroutines.CompletableDeferred -import net.sergeych.mp_tools.globalDefer -import net.sergeych.mp_tools.globalLaunch - -/** - * Naive source-based pacman that compiles all sources first, before first import could be resolved. - * It supports imports between [sources] but does not resolve nor detect cyclic imports which - * are not supported. - */ -class InlineSourcesPacman(pacman: Pacman, val sources: List) : Pacman(pacman) { - - data class Entry(val source: Source, val deferredModule: CompletableDeferred = CompletableDeferred()) - - - private val modules = run { - val result = mutableMapOf() - for (source in sources) { - val name = source.extractPackageName() - result[name] = Entry(source) - } - val inner = InitMan() - - for ((name, entry) in result) { - globalLaunch { - val scope = ModuleScope(inner, entry.source.startPos, name) - Compiler.compile(entry.source, inner).execute(scope) - entry.deferredModule.complete(scope) - } - } - result - } - - inner class InitMan : Pacman(parent) { - override suspend fun createModuleScope(name: String): ModuleScope? = modules[name]?.deferredModule?.await() - } - - private val readyModules by lazy { - globalDefer { - modules.entries.map { it.key to it.value.deferredModule.await() }.toMap() - } - } - - - override suspend fun createModuleScope(name: String): ModuleScope? = - readyModules.await()[name] -} \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt index db2396c..ef756bf 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt @@ -1,33 +1,47 @@ package net.sergeych.lyng +import net.sergeych.lyng.pacman.ImportProvider + /** - * Module scope supports importing and contains the [pacman]; it should be the same + * Module scope supports importing and contains the [importProvider]; it should be the same * used in [Compiler]; */ class ModuleScope( - val pacman: Pacman, + val importProvider: ImportProvider, pos: Pos = Pos.builtIn, - val packageName: String -) : Scope(pacman.rootScope, Arguments.EMPTY, pos) { + override val packageName: String +) : Scope(importProvider.rootScope, Arguments.EMPTY, pos) { - constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName) - - override suspend fun checkImport(pos: Pos, name: String, symbols: Map?) { - pacman.prepareImport(pos, name, symbols) - } + constructor(importProvider: ImportProvider, source: Source) : this(importProvider, source.startPos, source.fileName) /** - * Import symbols into the scope. It _is called_ after the module is imported by [checkImport]. - * If [checkImport] was not called, the symbols will not be imported with exception as module is not found. + * Import symbols into the scope. It _is called_ after the module is imported by [ImportProvider.prepareImport] + * which checks symbol availability and accessibility prior to execution. + * @param scope where to copy symbols from this module + * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values */ - override suspend fun importInto(scope: Scope, name: String, symbols: Map?) { - pacman.performImport(scope, name, symbols) + override suspend fun importInto(scope: Scope, symbols: Map?) { + val symbolsToImport = symbols?.keys?.toMutableSet() + for ((symbol, record) in this.objects) { + if (record.visibility.isPublic) { + val newName = symbols?.let { ss: Map -> + ss[symbol] + ?.also { symbolsToImport!!.remove(it) } + ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached") + } ?: symbol + if (newName in scope.objects) + scope.raiseError("symbol $newName already exists, redefinition on import is not allowed") + scope.objects[newName] = record + } + } + if (!symbolsToImport.isNullOrEmpty()) + scope.raiseSymbolNotFound("symbols $packageName.{$symbolsToImport} are.were not found") } - val packageNameObj by lazy { ObjString(packageName).asReadonly} + val packageNameObj by lazy { ObjString(packageName).asReadonly } override fun get(name: String): ObjRecord? { - return if( name == "__PACKAGE__") + return if (name == "__PACKAGE__") packageNameObj else super.get(name) diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt deleted file mode 100644 index 4217e96..0000000 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Pacman.kt +++ /dev/null @@ -1,69 +0,0 @@ -package net.sergeych.lyng - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock - -/** - * Package manager. Chained manager, too simple. Override [createModuleScope] to return either - * valid [ModuleScope] or call [parent] - or return null. - */ -abstract class Pacman( - val parent: Pacman? = null, - val rootScope: Scope = parent!!.rootScope, - val securityManager: SecurityManager = parent!!.securityManager -) { - private val opScopes = Mutex() - - private val cachedScopes = mutableMapOf() - - /** - * Create a new module scope if this pacman can import the module, return null otherwise so - * the manager can decide what to do - */ - abstract suspend fun createModuleScope(name: String): ModuleScope? - - suspend fun prepareImport(pos: Pos, name: String, symbols: Map?) { - if (!securityManager.canImportModule(name)) - throw ImportException(pos, "Module $name is not allowed") - symbols?.keys?.forEach { - if (!securityManager.canImportSymbol(name, it)) throw ImportException( - pos, - "Symbol $name.$it is not allowed" - ) - } - // if we can import the module, cache it, or go further - opScopes.withLock { - cachedScopes[name] ?: createModuleScope(name)?.let { cachedScopes[name] = it } - } ?: parent?.prepareImport(pos, name, symbols) ?: throw ImportException(pos, "Module $name is not found") - } - - suspend fun performImport(scope: Scope, name: String, symbols: Map?) { - val module = opScopes.withLock { cachedScopes[name] } - ?: scope.raiseSymbolNotFound("module $name not found") - val symbolsToImport = symbols?.keys?.toMutableSet() - for ((symbol, record) in module.objects) { - if (record.visibility.isPublic) { - val newName = symbols?.let { ss: Map -> - ss[symbol] - ?.also { symbolsToImport!!.remove(it) } - ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached") - } ?: symbol - if (newName in scope.objects) - scope.raiseError("symbol $newName already exists, redefinition on import is not allowed") - scope.objects[newName] = record - } - } - if (!symbolsToImport.isNullOrEmpty()) - scope.raiseSymbolNotFound("symbols $name.{$symbolsToImport} are.were not found") - } - - companion object { - val emptyAllowAll = object : Pacman(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) { - override suspend fun createModuleScope(name: String): ModuleScope? { - return null - } - - } - } - -} \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt index 24a6d5b..b161fc3 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt @@ -1,5 +1,7 @@ package net.sergeych.lyng +import net.sergeych.lyng.pacman.ImportProvider + /** * Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes. * Each block usually creates a scope. Accessing Lyng closures usually is done via a scope. @@ -16,6 +18,8 @@ open class Scope( var thisObj: Obj = ObjVoid, var skipScopeCreation: Boolean = false, ) { + open val packageName: String = "" + constructor( args: Arguments = Arguments.EMPTY, pos: Pos = Pos.builtIn, @@ -137,16 +141,17 @@ open class Scope( suspend fun eval(source: Source): Obj = Compiler.compile( source, - (this as? ModuleScope)?.pacman ?: Pacman.emptyAllowAll - ).execute(this) + (this as? ModuleScope)?.importProvider ?: ImportProvider.emptyAllowAll + ).execute(this) fun containsLocal(name: String): Boolean = name in objects - open suspend fun checkImport(pos: Pos, name: String, symbols: Map? = null) { - throw ImportException(pos, "Import is not allowed here: $name") - } - - open suspend fun importInto(scope: Scope, name: String, symbols: Map? = null) { - scope.raiseError(ObjIllegalOperationException(scope,"Import is not allowed here: import $name")) + /** + * Some scopes can be imported into other scopes, like [ModuleScope]. Those must correctly implement this method. + * @param scope where to copy symbols from this module + * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values + */ + open suspend fun importInto(scope: Scope, symbols: Map? = null) { + scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName")) } } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt new file mode 100644 index 0000000..ca77328 --- /dev/null +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt @@ -0,0 +1,48 @@ +package net.sergeych.lyng.pacman + +import net.sergeych.lyng.* + +/** + * Package manager INTERFACE (abstract class). Performs import routines + * using abstract [createModuleScope] method ot be implemented by heirs. + * + * Notice that [createModuleScope] is responsible for caching the modules; + * base class relies on caching. This is not implemented here as the correct + * caching strategy depends on the import provider + */ +abstract class ImportProvider( + val rootScope: Scope = Script.defaultScope, + val securityManager: SecurityManager = SecurityManager.allowAll +) { + /** + * Find an import and create a scope for it. This method must implement caching so repeated + * imports are not repeatedly loaded and parsed and should be cheap. + * + * @throws ImportException if the module is not found + */ + abstract suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope + + /** + * Check that the import is possible and allowed. This method is called on compile time by [Compiler]; + * actual module loading is performed by [ModuleScope.importInto] + */ + suspend fun prepareImport(pos: Pos, name: String, symbols: Map?): ModuleScope { + if (!securityManager.canImportModule(name)) + throw ImportException(pos, "Module $name is not allowed") + symbols?.keys?.forEach { + if (!securityManager.canImportSymbol(name, it)) throw ImportException( + pos, + "Symbol $name.$it is not allowed" + ) + } + return createModuleScope(pos, name) + } + + companion object { + val emptyAllowAll = object : ImportProvider(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) { + override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope { + throw ImportException(pos, "Empty import provider can't be used directly") + } + } + } +} \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt new file mode 100644 index 0000000..a9e004a --- /dev/null +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt @@ -0,0 +1,63 @@ +package net.sergeych.lyng.pacman + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import net.sergeych.lyng.* + +/** + * The import provider that imports sources available in memory. + */ +class InlineSourcesImportProvider(val sources: List, + rootScope: ModuleScope = ModuleScope(emptyAllowAll, Source.builtIn), + securityManager: SecurityManager = SecurityManager.allowAll +) : ImportProvider(rootScope, securityManager) { + + private class Entry( + val source: Source, + var scope: ModuleScope? = null, + ) + + private val inner = InitMan() + + private val modules = run { + val result = mutableMapOf() + for (source in sources) { + val name = source.extractPackageName() + result[name] = Entry(source) + } + result + } + + private var access = Mutex() + + /** + * Inner provider does not lock [access], the only difference; it is meant to be used + * exclusively by the coroutine that starts actual import chain + */ + private inner class InitMan : ImportProvider() { + override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope { + return doImport(packageName, pos) + } + } + + /** + * External interface, thread-safe. Can suspend until actual import is done. implements caching. + */ + override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope = + access.withLock { + doImport(packageName, pos) + } + + /** + * Perform actual import or return ready scope. __It must only be called when + * [access] is locked__, e.g. only internally + */ + private suspend fun doImport(packageName: String, pos: Pos): ModuleScope { + modules[packageName]?.scope?.let { return it } + val entry = modules[packageName] ?: throw ImportException(pos, "Unknown package $packageName") + return ModuleScope(inner, pos, packageName).apply { + entry.scope = this + eval(entry.source) + } + } +} \ No newline at end of file diff --git a/lynglib/src/commonTest/kotlin/ScriptTest.kt b/lynglib/src/commonTest/kotlin/ScriptTest.kt index c2396cd..153b122 100644 --- a/lynglib/src/commonTest/kotlin/ScriptTest.kt +++ b/lynglib/src/commonTest/kotlin/ScriptTest.kt @@ -1,8 +1,8 @@ - import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList import kotlinx.coroutines.test.runTest import net.sergeych.lyng.* +import net.sergeych.lyng.pacman.InlineSourcesImportProvider import kotlin.test.* class ScriptTest { @@ -2250,18 +2250,21 @@ class ScriptTest { @Test fun testLet() = runTest { - eval(""" + eval( + """ class Point(x=0,y=0) assert( Point() is Object) Point().let { println(it.x, it.y) } val x = null x?.let { println(it.x, it.y) } - """.trimIndent()) + """.trimIndent() + ) } @Test fun testApply() = runTest { - eval(""" + eval( + """ class Point(x,y) // see the difference: apply changes this to newly created Point: val p = Point(1,2).apply { @@ -2270,12 +2273,14 @@ class ScriptTest { assertEquals(p, Point(2,3)) >>> void - """.trimIndent()) + """.trimIndent() + ) } @Test fun testApplyThis() = runTest { - eval(""" + eval( + """ class Point(x,y) // see the difference: apply changes this to newly created Point: val p = Point(1,2).apply { @@ -2284,12 +2289,14 @@ class ScriptTest { assertEquals(p, Point(2,3)) >>> void - """.trimIndent()) + """.trimIndent() + ) } @Test fun testExtend() = runTest() { - eval(""" + eval( + """ fun Int.isEven() { this % 2 == 0 @@ -2311,7 +2318,8 @@ class ScriptTest { assert( 12.1.isInteger() == false ) assert( "5".isInteger() ) assert( ! "5.2".isInteger() ) - """.trimIndent()) + """.trimIndent() + ) } @Test @@ -2319,18 +2327,20 @@ class ScriptTest { val c = Scope() val arr = c.eval("[1,2,3]") // array is iterable so we can: - assertEquals(listOf(1,2,3), arr.toFlow(c).map { it.toInt() }.toList()) + assertEquals(listOf(1, 2, 3), arr.toFlow(c).map { it.toInt() }.toList()) } @Test fun testAssociateBy() = runTest() { - eval(""" + eval( + """ val m = [123, 456].associateBy { "k:%s"(it) } println(m) assertEquals(123, m["k:123"]) assertEquals(456, m["k:456"]) - """) - listOf(1,2,3).associateBy { it * 10 } + """ + ) + listOf(1, 2, 3).associateBy { it * 10 } } // @Test @@ -2354,7 +2364,7 @@ class ScriptTest { fun foo() { "foo1" } """.trimIndent() - val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc))) + val pm = InlineSourcesImportProvider(listOf(Source("foosrc", foosrc))) val src = """ import lyng.foo @@ -2380,11 +2390,12 @@ class ScriptTest { fun bar() { "bar1" } """.trimIndent() - val pm = InlineSourcesPacman( - Pacman.emptyAllowAll, listOf( + val pm = InlineSourcesImportProvider( + listOf( Source("barsrc", barsrc), Source("foosrc", foosrc), - )) + ) + ) val src = """ import lyng.foo @@ -2396,4 +2407,37 @@ class ScriptTest { assertEquals("foo1 / bar1", scope.eval(src).toString()) } + @Test + fun testImportsCircular() = runTest { + val foosrc = """ + package lyng.foo + + import lyng.bar + + fun foo() { "foo1" } + """.trimIndent() + val barsrc = """ + package lyng.bar + + import lyng.foo + + fun bar() { "bar1" } + """.trimIndent() + val pm = InlineSourcesImportProvider( + listOf( + Source("barsrc", barsrc), + Source("foosrc", foosrc), + ) + ) + + val src = """ + import lyng.bar + + foo() + " / " + bar() + """.trimIndent().toSource("test") + + val scope = ModuleScope(pm, src) + assertEquals("foo1 / bar1", scope.eval(src).toString()) + } + } \ No newline at end of file diff --git a/lynglib/src/jvmTest/kotlin/OtherTests.kt b/lynglib/src/jvmTest/kotlin/OtherTests.kt index dab092a..890120d 100644 --- a/lynglib/src/jvmTest/kotlin/OtherTests.kt +++ b/lynglib/src/jvmTest/kotlin/OtherTests.kt @@ -1,6 +1,9 @@ import junit.framework.TestCase.assertEquals import kotlinx.coroutines.runBlocking -import net.sergeych.lyng.* +import net.sergeych.lyng.ModuleScope +import net.sergeych.lyng.Source +import net.sergeych.lyng.pacman.InlineSourcesImportProvider +import net.sergeych.lyng.toSource import kotlin.test.Test class OtherTests { @@ -18,8 +21,8 @@ class OtherTests { fun bar() { "bar1" } """.trimIndent() - val pm = InlineSourcesPacman( - Pacman.emptyAllowAll, listOf( + val pm = InlineSourcesImportProvider( + listOf( Source("foosrc", foosrc), Source("barsrc", barsrc), ))