From 26282d3e22a59429ae55a2eec15c5801ec0874b1 Mon Sep 17 00:00:00 2001 From: sergeych Date: Wed, 9 Jul 2025 13:15:28 +0300 Subject: [PATCH] fix #38 ImportManager integrated into Scope tree and all systems --- docs/samples/helloworld.lyng | 2 +- lynglib/build.gradle.kts | 2 +- .../net/sergeych/lyng/ArgsDeclaration.kt | 4 +- .../kotlin/net/sergeych/lyng/Compiler.kt | 6 +- .../kotlin/net/sergeych/lyng/Parser.kt | 10 +- .../kotlin/net/sergeych/lyng/Scope.kt | 32 ++++- .../kotlin/net/sergeych/lyng/Script.kt | 11 +- .../net/sergeych/lyng/pacman/ImportManager.kt | 126 ++++++++++++++++++ .../sergeych/lyng/pacman/ImportProvider.kt | 18 +-- .../pacman/InlineSourcesImportProvider.kt | 68 +++------- lynglib/src/commonTest/kotlin/ScriptTest.kt | 22 ++- 11 files changed, 224 insertions(+), 77 deletions(-) create mode 100644 lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt diff --git a/docs/samples/helloworld.lyng b/docs/samples/helloworld.lyng index f22731c..2a89faa 100755 --- a/docs/samples/helloworld.lyng +++ b/docs/samples/helloworld.lyng @@ -1,3 +1,3 @@ #!/bin/env jlyng -println("Hello, world2! "+ARGV); +println("Hello, world!"); diff --git a/lynglib/build.gradle.kts b/lynglib/build.gradle.kts index 072b795..ab1130c 100644 --- a/lynglib/build.gradle.kts +++ b/lynglib/build.gradle.kts @@ -4,7 +4,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl import org.jetbrains.kotlin.gradle.dsl.JvmTarget group = "net.sergeych" -version = "0.7.1-SNAPSHOT" +version = "0.7.2-SNAPSHOT" buildscript { repositories { diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt index 0289af5..798899a 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt @@ -54,8 +54,8 @@ data class ArgsDeclaration(val params: List, val endTokenType: Token.Type) i < callArgs.size -> callArgs[i] a.defaultValue != null -> a.defaultValue.execute(scope) else -> { - println("callArgs: ${callArgs.joinToString()}") - println("tailBlockMode: ${arguments.tailBlockMode}") +// println("callArgs: ${callArgs.joinToString()}") +// println("tailBlockMode: ${arguments.tailBlockMode}") scope.raiseIllegalArgument("too few arguments for the call") } } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt index 951b194..eedf3de 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt @@ -7,7 +7,7 @@ import net.sergeych.lyng.pacman.ImportProvider */ class Compiler( val cc: CompilerContext, - val importProvider: ImportProvider = ImportProvider.emptyAllowAll, + val importManager: ImportProvider = Script.defaultImportManager, @Suppress("UNUSED_PARAMETER") settings: Settings = Settings() ) { @@ -43,7 +43,7 @@ class Compiler( cc.next() val pos = cc.currentPos() val name = loadQualifiedName() - val module = importProvider.prepareImport(pos, name, null) + val module = importManager.prepareImport(pos, name, null) statements += statement { module.importInto(this, null) ObjVoid @@ -1620,7 +1620,7 @@ class Compiler( companion object { - suspend fun compile(source: Source, importProvider: ImportProvider = ImportProvider.emptyAllowAll): Script { + suspend fun compile(source: Source, importProvider: ImportProvider = Script.defaultImportManager): Script { return Compiler(CompilerContext(parseLyng(source)),importProvider).parseScript() } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt index 2f6f1f8..87fa4a5 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt @@ -109,7 +109,7 @@ private class Parser(fromPos: Pos) { '/' -> when (currentChar) { '/' -> { pos.advance() - Token(loadToEnd().trim(), from, Token.Type.SINLGE_LINE_COMMENT) + Token(loadToEndOfLine().trim(), from, Token.Type.SINLGE_LINE_COMMENT) } '*' -> { @@ -445,7 +445,7 @@ private class Parser(fromPos: Pos) { } } - private fun loadToEnd(): String { + private fun loadToEndOfLine(): String { val result = StringBuilder() val l = pos.line do { @@ -479,4 +479,10 @@ private class Parser(fromPos: Pos) { return null } + init { + // skip shebang + if( pos.readFragment("#!") ) + loadToEndOfLine() + } + } \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt index b161fc3..56bb037 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt @@ -1,15 +1,20 @@ package net.sergeych.lyng -import net.sergeych.lyng.pacman.ImportProvider +import net.sergeych.lyng.pacman.ImportManager /** * Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes. * Each block usually creates a scope. Accessing Lyng closures usually is done via a scope. * - * There are special types of scopes: + * To create default scope, use default `Scope()` constructor, it will create a scope with a parent + * module scope with default [ImportManager], you can access with [importManager] as needed. * - * - [Script.defaultScope] - root scope for a script, safe one - * - [AppliedScope] - scope used to apply a closure to some thisObj scope + * If you want to create [ModuleScope] by hand, try [importManager] and [ImportManager.newModule], + * or [ImportManager.newModuleAt]. + * + * There are special types of scopes: + * + * - [AppliedScope] - scope used to apply a closure to some thisObj scope */ open class Scope( val parent: Scope?, @@ -24,7 +29,7 @@ open class Scope( args: Arguments = Arguments.EMPTY, pos: Pos = Pos.builtIn, ) - : this(Script.defaultScope, args, pos) + : this(Script.defaultImportManager.newModuleAt(pos), args, pos) fun raiseNotImplemented(what: String = "operation"): Nothing = raiseError("$what is not implemented") @@ -141,7 +146,7 @@ open class Scope( suspend fun eval(source: Source): Obj = Compiler.compile( source, - (this as? ModuleScope)?.importProvider ?: ImportProvider.emptyAllowAll + (this as? ModuleScope)?.importProvider ?: Script.defaultImportManager ).execute(this) fun containsLocal(name: String): Boolean = name in objects @@ -154,4 +159,19 @@ open class Scope( open suspend fun importInto(scope: Scope, symbols: Map? = null) { scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName")) } + + /** + * Find a first [ImportManager] in this Scope hierarchy. Normally there should be one. Found instance is cached. + * + * Use it to register your package sources, see [ImportManager] features. + * + * @throws IllegalStateException if there is no such manager (if you create some specific scope with no manager, + * then you knew what you did) + */ + val importManager: ImportManager by lazy { + if( this is ModuleScope ) + (importProvider as? ImportManager)?.let { return@lazy it } + parent?.importManager ?: throw IllegalStateException("this scope has no manager in the chain") + } + } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt index 52f1113..8804139 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt @@ -1,6 +1,7 @@ package net.sergeych.lyng import kotlinx.coroutines.delay +import net.sergeych.lyng.pacman.ImportManager import kotlin.math.* class Script( @@ -17,10 +18,11 @@ class Script( return lastResult } - suspend fun execute() = execute(defaultScope.copy(pos = pos)) + suspend fun execute() = execute(defaultImportManager.newModule()) companion object { - val defaultScope: Scope = Scope().apply { + + private val rootScope: Scope = Scope(null).apply { ObjException.addExceptionsToContext(this) addFn("println") { for ((i, a) in args.withIndex()) { @@ -170,8 +172,9 @@ class Script( getOrCreateNamespace("Math").apply { addConst("PI", pi) } - - } + + val defaultImportManager: ImportManager by lazy { ImportManager(rootScope, SecurityManager.allowAll) } + } } \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt new file mode 100644 index 0000000..a5c0c73 --- /dev/null +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt @@ -0,0 +1,126 @@ +package net.sergeych.lyng.pacman + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import net.sergeych.lyng.* + +/** + * Import manager allow to register packages with builder lambdas and act as an + * [ImportProvider]. Note that packages _must be registered_ first with [addPackage], + * [addSourcePackages] or [addTextPackages]. Registration is cheap, actual package + * building is lazily performed on [createModuleScope], when the package will + * be first imported. + * + * It is possible to register new packages at any time, but it is not allowed to override + * packages already registered. + */ +class ImportManager( + rootScope: Scope = Script.defaultImportManager.newModule(), + securityManager: SecurityManager = SecurityManager.allowAll +): ImportProvider(rootScope, securityManager) { + + private inner class Entry( + val packageName: String, + val builder: suspend (ModuleScope) -> Unit, + var cachedScope: ModuleScope? = null + ) { + + suspend fun getScope(pos: Pos): ModuleScope { + cachedScope?.let { return it } + return ModuleScope(inner, pos, packageName).apply { + cachedScope = this + builder(this) + } + } + } + + + /** + * Inner provider does not lock [access], the only difference; it is meant to be used + * exclusively by the coroutine that starts actual import chain + */ + private inner class InternalProvider : ImportProvider(rootScope) { + override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope { + return doImport(packageName, pos) + } + } + + /** + * Inner module import provider used to prepare lazily prepared modules + */ + private val inner = InternalProvider() + + + private val imports = mutableMapOf() + private val access = Mutex() + + /** + * Register new package that can be imported. It is not possible to unregister or + * update package already registered. + * + * Packages are lazily created when first imported somewhere, so the registration is + * cheap; the recommended procedure is to register all available packages prior to + * compile with this. + * + * @param name package name + * @param builder lambda to create actual package using the given [ModuleScope] + */ + suspend fun addPackage(name: String, builder: suspend (ModuleScope) -> Unit) { + access.withLock { + if (name in imports) + throw IllegalArgumentException("Package $name already exists") + imports[name] = Entry(name, builder) + } + } + + /** + * Bulk [addPackage] with slightly better performance + */ + @Suppress("unused") + suspend fun addPackages(registrationData: List Unit>>) { + access.withLock { + for (pp in registrationData) { + if (pp.first in imports) + throw IllegalArgumentException("Package ${pp.first} already exists") + imports[pp.first] = Entry(pp.first, pp.second) + } + } + } + + /** + * Perform actual import or return ready scope. __It must only be called when + * [access] is locked__, e.g. only internally + */ + private suspend fun doImport(packageName: String, pos: Pos): ModuleScope { + val entry = imports[packageName] ?: throw ImportException(pos, "package not found: $packageName") + return entry.getScope(pos) + } + + override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope = + doImport(packageName, pos) + + /** + * Add packages that only need to compile [Source]. + */ + suspend fun addSourcePackages(vararg sources: Source) { + for( s in sources) { + addPackage(s.extractPackageName()) { + it.eval(s) + } + + } + } + + /** + * Add source packages using package name as [Source.fileName], for simplicity + */ + suspend fun addTextPackages(vararg sourceTexts: String) { + for( s in sourceTexts) { + var source = Source("tmp", s) + val packageName = source.extractPackageName() + source = Source(packageName, s) + addPackage(packageName) { it.eval(source)} + } + } + +} \ No newline at end of file diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt index ca77328..b7ae739 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt @@ -11,7 +11,7 @@ import net.sergeych.lyng.* * caching strategy depends on the import provider */ abstract class ImportProvider( - val rootScope: Scope = Script.defaultScope, + val rootScope: Scope, val securityManager: SecurityManager = SecurityManager.allowAll ) { /** @@ -38,11 +38,11 @@ abstract class ImportProvider( return createModuleScope(pos, name) } - companion object { - val emptyAllowAll = object : ImportProvider(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) { - override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope { - throw ImportException(pos, "Empty import provider can't be used directly") - } - } - } -} \ No newline at end of file + fun newModule() = newModuleAt(Pos.builtIn) + + fun newModuleAt(pos: Pos): ModuleScope = + ModuleScope(this, pos, "unknown") +} + + + diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt index a9e004a..e59ac5b 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt @@ -1,63 +1,35 @@ package net.sergeych.lyng.pacman -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.CompletableDeferred import net.sergeych.lyng.* +import net.sergeych.mp_tools.globalLaunch /** - * The import provider that imports sources available in memory. + * The sample import provider that imports sources available in memory. + * on construction time. + * + * Actually it is left here only as a demo. */ -class InlineSourcesImportProvider(val sources: List, - rootScope: ModuleScope = ModuleScope(emptyAllowAll, Source.builtIn), +class InlineSourcesImportProvider(sources: List, + rootScope: ModuleScope = Script.defaultImportManager.newModule(), securityManager: SecurityManager = SecurityManager.allowAll -) : ImportProvider(rootScope, securityManager) { +) : ImportProvider(rootScope) { - private class Entry( - val source: Source, - var scope: ModuleScope? = null, - ) + private val manager = ImportManager(rootScope, securityManager) - private val inner = InitMan() + private val readyManager = CompletableDeferred() - private val modules = run { - val result = mutableMapOf() - for (source in sources) { - val name = source.extractPackageName() - result[name] = Entry(source) - } - result + /** + * This implementation only + */ + override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope { + return readyManager.await().createModuleScope(pos, packageName) } - private var access = Mutex() - - /** - * Inner provider does not lock [access], the only difference; it is meant to be used - * exclusively by the coroutine that starts actual import chain - */ - private inner class InitMan : ImportProvider() { - override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope { - return doImport(packageName, pos) - } - } - - /** - * External interface, thread-safe. Can suspend until actual import is done. implements caching. - */ - override suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope = - access.withLock { - doImport(packageName, pos) - } - - /** - * Perform actual import or return ready scope. __It must only be called when - * [access] is locked__, e.g. only internally - */ - private suspend fun doImport(packageName: String, pos: Pos): ModuleScope { - modules[packageName]?.scope?.let { return it } - val entry = modules[packageName] ?: throw ImportException(pos, "Unknown package $packageName") - return ModuleScope(inner, pos, packageName).apply { - entry.scope = this - eval(entry.source) + init { + globalLaunch { + manager.addSourcePackages(*sources.toTypedArray()) + readyManager.complete(manager) } } } \ No newline at end of file diff --git a/lynglib/src/commonTest/kotlin/ScriptTest.kt b/lynglib/src/commonTest/kotlin/ScriptTest.kt index 153b122..2a757e5 100644 --- a/lynglib/src/commonTest/kotlin/ScriptTest.kt +++ b/lynglib/src/commonTest/kotlin/ScriptTest.kt @@ -604,7 +604,7 @@ class ScriptTest { } @Test - fun testAssignArgumentsmiddleEllipsis() = runTest { + fun testAssignArgumentsMiddleEllipsis() = runTest { val ttEnd = Token.Type.RBRACE val pa = ArgsDeclaration( listOf( @@ -2440,4 +2440,24 @@ class ScriptTest { assertEquals("foo1 / bar1", scope.eval(src).toString()) } + @Test + fun testDefaultImportManager() = runTest { + val scope = Scope() + assertFails { + scope.eval(""" + import foo + foo() + """.trimIndent()) + } + scope.importManager.addTextPackages(""" + package foo + + fun foo() { "bar" } + """.trimIndent()) + scope.eval(""" + import foo + assertEquals( "bar", foo()) + """.trimIndent()) + } + } \ No newline at end of file