refs #35 generic implementation of Huffman compression for variable bit length source alphabet

added clean import scope Scope.new()

bin/local_jrelease

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+root=./lyng/build/install/lyng-jvm/
+
+./gradlew :lyng:installJvmDist
+#strip $file
+#upx   $file
+rm -rf ~/bin/jlyng-jvm || true
+rm ~/bin/jlyng 2>/dev/null || true
+mkdir -p ~/bin/jlyng-jvm
+cp -R $root ~/bin/jlyng-jvm
+ln -s ~/bin/jlyng-jvm/lyng-jvm/bin/lyng ~/bin/jlyng

docs/Buffer.md

@@ -0,0 +1,121 @@
+# Binary `Buffer`
+
+Buffers are effective unsigned byte arrays of fixed size. Buffers content is mutable,
+unlike its size. Buffers are comparable and implement [Array], thus [Collection] and [Iterable]. Buffer iterators return
+its contents as unsigned bytes converted to `Int`
+
+Buffers needs to be imported with `import lyng.buffer`:
+
+    import lyng.buffer
+
+    assertEquals(5, Buffer("Hello").size)
+    >>> void
+
+Buffer is _immutable_, there is a `MutableBuffer` with same interface but mutable.
+
+## Constructing
+
+There are a lo of ways to construct a buffer:
+
+    import lyng.buffer
+    
+    // from string using utf8 encoding:
+    assertEquals( 5, Buffer("hello").size )
+
+    // from bytes, e.g. integers in range 0..255
+    assertEquals( 255, Buffer(1,2,3,255).last() )
+
+    // from whatever iterable that produces bytes, e.g.
+    // integers in 0..255 range:
+    assertEquals( 129, Buffer([1,2,129]).last() )
+
+    // Empty buffer of fixed size:
+    assertEquals(100, Buffer(100).size)
+    assertEquals(0, Buffer(100)[0])
+
+    // Note that you can use list iteral to create buffer with 1 byte:
+    assertEquals(1, Buffer([100]).size)
+    assertEquals(100, Buffer([100])[0])
+
+    >>> void
+
+## Accessing and modifying
+
+Buffer implement [Array] and therefore can be accessed, and `MutableBuffers` also modified:
+
+    import lyng.buffer
+    val b1 = Buffer( 1, 2, 3)
+    assertEquals( 2, b1[1] )
+
+    val b2 = b1.toMutable()
+    assertEquals( 2, b1[1] )
+    b2[1]++
+    b2[0] = 100
+    assertEquals( Buffer(100, 3, 3), b2)
+
+    // b2 is a mutable copy so b1 has not been changed:
+    assertEquals( Buffer(1, 2, 3), b1)
+
+    >>> void
+
+Buffer provides concatenation with another Buffer:
+
+    import lyng.buffer
+    val b = Buffer(101, 102)
+    assertEquals( Buffer(101, 102, 1, 2), b + [1,2])
+    >>> void
+
+Please note that indexed bytes are _readonly projection_, e.g. you can't modify these with
+
+## Comparing
+
+Buffers are comparable with other buffers (and notice there are _mutable_ buffers, bu default buffers ar _immutable_):
+
+    import lyng.buffer
+    val b1 = Buffer(1, 2, 3)
+    val b2 = Buffer(1, 2, 3)
+    val b3 = MutableBuffer(b2)
+    
+    b3[0] = 101
+
+    assert( b3 > b1 )
+    assert( b2 == b1 )
+    // longer string with same prefix is considered bigger:
+    assert( b2 + "!".characters() > b1 )
+    // note that characters() provide Iterable of characters that 
+    // can be concatenated to Buffer
+
+    >>> void
+
+## Slicing
+
+As with [List], it is possible to use ranges as indexes to slice a Buffer:
+
+    import lyng.buffer
+
+    val a = Buffer( 100, 101, 102, 103, 104, 105 )
+    assertEquals( a[ 0..1 ], Buffer(100, 101) )
+    assertEquals( a[ 0 ..< 2 ], Buffer(100, 101) )
+    assertEquals( a[  ..< 2 ], Buffer(100, 101) )
+    assertEquals( a[  4.. ], Buffer(104, 105) )
+    assertEquals( a[  2..3 ], Buffer(102, 103) )
+
+    >>> void
+
+## Members
+
+| name          | meaning                            | type          |
+|---------------|------------------------------------|---------------|
+| `size`        | size                               | Int           |
+| `decodeUtf8`  | decodee to String using UTF8 rules | Any           |
+| `+`           | buffer concatenation               | Any           |
+| `toMutable()` | create a mutable copy              | MutableBuffer |
+
+(1)
+: optimized implementation that override `Iterable` one
+
+Also, it inherits methods from [Iterable] and [Array].
+
+
+[Range]: Range.md
+[Iterable]: Iterable.md

docs/Map.md

@@ -3,6 +3,8 @@
 Map is a mutable collection of key-value pars, where keys are unique. Maps could be created with
 constructor or `.toMap` methods. When constructing from a list, each list item must be a [Collection] with exactly 2 elements, for example, a [List].
 
+Important thing is that maps can't contain `null`: it is used to return from missing elements.
+
 Constructed map instance is of class `Map` and implements `Collection` (and therefore `Iterable`)
 
     val map = Map( ["foo", 1], ["bar", "buzz"] )
@@ -16,7 +18,7 @@ Map keys could be any objects (hashable, e.g. with reasonable hashCode, most of
     val map = Map( ["foo", 1], ["bar", "buzz"], [42, "answer"] )
     assert( map["bar"] == "buzz")
     assert( map[42] == "answer" )
-    assertThrows { map["nonexistent"] }
+    assertEquals( null, map["nonexisting"])
     assert( map.getOrNull(101) == null )
     assert( map.getOrPut(911) { "nine-eleven" } == "nine-eleven" )
     // now 91 entry is set:

docs/OOP.md

@@ -250,11 +250,10 @@ Note `Real` class: it is global variable for Real class; there are such class in
     assert('$'::class == Char)
     >>> void
 
-More complex is singleton classes, because you don't need to compare their class
-instances and generally don't need them at all, these are normally just Obj:
+Singleton classes also have class:
 
     null::class
-    >>> Obj
+    >>> Null
 
 At this time, `Obj` can't be accessed as a class.
 

docs/samples/helloworld.lyng

@@ -1,2 +1,3 @@
+#!/bin/env jlyng
 
 println("Hello, world!");

docs/time.md

@@ -0,0 +1,215 @@
+# Lyng time functions
+
+Lyng date and time support requires importing `lyng.time` packages. Lyng uses simple yet modern time object models:
+
+- `Instant` class for time stamps with platform-dependent resolution
+- `Duration` to represent amount of time not depending on the calendar, e.g. in absolute units (milliseconds, seconds,
+  hours, days)
+
+## Time instant: `Instant`
+
+Represent some moment of time not depending on the calendar (calendar for example may b e changed, daylight saving time
+can be for example introduced or dropped). It is similar to `TIMESTAMP` in SQL or `Instant` in Kotlin. Some moment of
+time; not the calendar date.
+
+Instant is comparable to other Instant. Subtracting instants produce `Duration`, period in time that is not dependent on
+the calendar, e.g. absolute time period.
+
+It is possible to add or subtract `Duration` to and from `Instant`, that gives another `Instant`.
+
+Instants are converted to and from `Real` number of seconds before or after Unix Epoch, 01.01.1970. Constructor with
+single number parameter constructs from such number of seconds,
+and any instance provide `.epochSeconds` member:
+
+    import lyng.time
+
+    // default constructor returns time now:
+    val t1 = Instant()
+    val t2 = Instant()
+    assert( t2 - t1 < 1.millisecond )
+    assert( t2.epochSeconds - t1.epochSeconds < 0.001 )
+    >>> void
+
+## Constructing
+
+    import lyng.time
+
+    // empty constructor gives current time instant using system clock:
+    val now = Instant()
+
+    // constructor with Instant instance makes a copy:
+    assertEquals( now, Instant(now) )
+
+    // constructing from a number is trated as seconds since unix epoch:
+    val copyOfNow = Instant( now.epochSeconds )
+
+    // note that instant resolution is higher that Real can hold
+    // so reconstructed from real slightly differs:
+    assert( abs( (copyOfNow - now).milliseconds ) < 0.01 )
+    >>> void
+
+The resolution of system clock could be more precise and double precision real number of `Real`, keep it in mind.
+
+## Comparing and calculating periods
+
+    import lyng.time
+    
+    val now = Instant()
+    
+    // you cam add or subtract periods, and compare
+    assert( now - 5.minutes < now )
+    val oneHourAgo = now - 1.hour
+    assertEquals( now, oneHourAgo + 1.hour)
+
+    >>> void
+
+## Getting the max precision
+
+Normally, subtracting instants gives precision to microseconds, which is well inside the jitter
+the language VM adds. Still `Instant()` or `Instant.now()` capture most precise system timer at hand and provide inner
+value of 12 bytes, up to nanoseconds (hopefully). To access it use:
+
+    import lyng.time
+
+    // capture time
+    val now = Instant.now()
+
+    // this is Int value, number of whole epoch 
+    // milliseconds to the moment, it fits 8 bytes Int well
+    val seconds = now.epochWholeSeconds
+    assert(seconds is Int)
+
+    // and this is Int value of nanoseconds _since_ the epochMillis,
+    // it effectively add 4 more mytes int:
+    val nanos = now.nanosecondsOfSecond
+    assert(nanos is Int)
+    assert( nanos in 0..999_999_999 )
+
+    // we can construct epochSeconds from these parts:
+    assertEquals( now.epochSeconds, nanos * 1e-9 + seconds )
+    >>> void
+
+## Truncating to more realistic precision
+
+Full precision Instant is way too long and impractical to store, especially when serializing,
+so it is possible to truncate it to milliseconds, microseconds or seconds:
+
+    import lyng.time
+    import lyng.serialization
+
+    // max supported size (now microseconds for serialized value):
+    // note that encoding return _bit array_ and this is a _bit size_:
+    val s0 = Lynon.encode(Instant.now()).size
+
+    // shorter: milliseconds only
+    val s1 = Lynon.encode(Instant.now().truncateToMillisecond()).size
+
+    // truncated to seconds, good for file mtime, etc:
+    val s2 = Lynon.encode(Instant.now().truncateToSecond()).size 
+    assert( s1 < s0 )
+    assert( s2 < s1 )
+    >>> void
+
+## Formatting instants
+
+You can freely use `Instant` in string formatting. It supports usual sprintf-style formats:
+
+        import lyng.time
+        val now = Instant()
+
+        // will be something like "now: 12:10:05"
+        val currentTimeOnly24 =  "now: %tT"(now)
+
+        // we can extract epoch second with formatting too,
+        // this was since early C time
+
+        // get epoch while seconds from formatting
+        val unixEpoch = "Now is %ts since unix epoch"(now)
+        
+        // and it is the same as now.epochSeconds, int part:
+        assertEquals( unixEpoch, "Now is %d since unix epoch"(now.epochSeconds.toInt()) )
+        >>> void
+
+See
+the [complete list of available formats](https://github.com/sergeych/mp_stools?tab=readme-ov-file#datetime-formatting)
+and the [formatting reference](https://github.com/sergeych/mp_stools?tab=readme-ov-file#printf--sprintf): it all works
+in Lyng as `"format"(args...)`!
+
+## Instant members
+
+| member                         | description                                             |
+|--------------------------------|---------------------------------------------------------|
+| epochSeconds: Real             | positive or negative offset in seconds since Unix epoch |
+| epochWholeSeconds: Int         | same, but in _whole seconds_. Slightly faster           |
+| nanosecondsOfSecond: Int       | offset from epochWholeSeconds in nanos (1)              |
+| isDistantFuture: Bool          | true if it `Instant.distantFuture`                      |
+| isDistantPast: Bool            | true if it `Instant.distantPast`                        |
+| truncateToSecond: Intant       | create new instnce truncated to second                  |  
+| truncateToMillisecond: Instant | truncate new instance with to millisecond               |
+| truncateToMicrosecond: Instant | truncate new instance to microsecond                    |
+
+(1)
+: The value of nanoseconds is to be added to `epochWholeSeconds` to get exact time point. It is in 0..999_999_999 range.
+The precise time instant value therefore needs as for now 12 bytes integer; we might use bigint later (it is planned to
+be added)
+
+## Class members
+
+| member                         | description                                  |
+|--------------------------------|----------------------------------------------|
+| Instant.now()                  | create new instance with current system time |
+| Instant.distantPast: Instant   | most distant instant in past                 |
+| Instant.distantFuture: Instant | most distant instant in future               |
+
+# `Duraion` class
+
+Represent absolute time distance between two `Instant`.
+
+    import lyng.time
+    val t1 = Instant()
+
+    // yes we can delay to period, and it is not blocking. is suspends!
+    delay(1.millisecond)
+
+    val t2 = Instant()
+    // be suspend, so actual time may vary:
+    assert( t2 - t1 >= 1.millisecond)
+    assert( t2 - t1 < 100.millisecond)
+    >>> void
+
+Duration can be converted from numbers, like `5.minutes` and so on. Extensions are created for
+`Int` and `Real`, so for n as Real or Int it is possible to create durations::
+
+- `n.millisecond`, `n.milliseconds`
+- `n.second`, `n.seconds`
+- `n.minute`, `n.minutes`
+- `n.hour`, `n.hours`
+- `n.day`, `n.days`
+
+The bigger time units like months or years are calendar-dependent and can't be used with `Duration`.
+
+Each duration instance can be converted to number of any of these time units, as `Real` number, if `d` is a `Duration`
+instance:
+
+- `d.microseconds`
+- `d.milliseconds`
+- `d.seconds`
+- `d.minutes`
+- `d.hours`
+- `d.days`
+
+for example
+
+    import lyng.time
+    assertEquals( 60, 1.minute.seconds )
+    assertEquals( 10.milliseconds, 0.01.seconds )
+
+    >>> void
+
+# Utility functions
+
+## delay(duration: Duration)
+
+Suspends current coroutine for at least the specified duration.
+
+

docs/tutorial.md

@@ -14,7 +14,7 @@ __Other documents to read__ maybe after this one:
 - [Advanced topics](advanced_topics.md), [declaring arguments](declaring_arguments.md)
 - [OOP notes](OOP.md), [exception handling](exceptions_handling.md)
 - [math in Lyng](math.md)
-- Some class references: [List], [Set], [Map], [Real], [Range], [Iterable], [Iterator]
+- Some class references: [List], [Set], [Map], [Real], [Range], [Iterable], [Iterator], [time manipulation](time.md)
 - Some samples: [combinatorics](samples/combinatorics.lyng.md), national vars and loops: [сумма ряда](samples/сумма_ряда.lyng.md). More at [samples folder](samples)
 
 # Expressions
@@ -530,7 +530,8 @@ The simplest way to concatenate lists is using `+` and `+=`:
     list += [2, 1]      
     // or you can append a single element:
     list += "end"
-    assert( list == [1, 2, 2, 1, "end"])
+    assertEquals( list, [1, 2, 2, 1, "end"])
+    void
     >>> void
 
 ***Important note***: the pitfall of using `+=` is that you can't append in [Iterable] instance as an object: it will always add all its contents. Use `list.add` to add a single iterable instance:
@@ -641,6 +642,11 @@ You can get ranges to extract a portion from a list:
     assertEquals( [2,3], list[1..<3])
     >>> void
 
+# Buffers
+
+[Buffer] is a special implementation of an [Array] of unsigned bytes, in the
+[separate file](Buffer.md).
+
 # Sets
 
 Set are unordered collection of unique elements, see [Set]. Sets are [Iterable] but have no indexing access.
@@ -662,7 +668,6 @@ are [Iterable]:
 
 Please see [Map] reference for detailed description on using Maps.
 
-
 # Flow control operators
 
 ## if-then-else
@@ -740,13 +745,13 @@ You can thest that _when expression_ is _contained_, or not contained, in some o
 
 Typical builtin types that are containers (e.g. support `conain`):
 
-| class      | notes                                      |
-|------------|--------------------------------------------|
-| Collection | contains an element (1)                    |
-| Array      | faster maybe that Collection's             |
-| List       | faster than Array's                        |
-| String     | character in string or substring in string |
-| Range      | object is included in the range (2)        |
+| class      | notes                                          |
+|------------|------------------------------------------------|
+| Collection | contains an element (1)                        |
+| Array      | faster maybe that Collection's                 |
+| List       | faster than Array's                            |
+| String     | character in string or substring in string (3) |
+| Range      | object is included in the range (2)            |
 
 (1)
 : Iterable is not the container as it can be infinite
@@ -760,6 +765,9 @@ Typical builtin types that are containers (e.g. support `conain`):
     assert( "x" !in 'a'..'z') // string in character range: could be error
     >>> void
 
+(3)
+: `String` also can provide array of characters directly with `str.characters()`, which is [Iterable] and [Array]. String itself is not iterable as otherwise it will interfere when adding strigns to lists (it will add _characters_ it it would be iterable).
+
 So we recommend not to mix characters and string ranges; use `ch in str` that works
 as expected:
 
@@ -1092,6 +1100,17 @@ and you can use ranges in for-loops:
 
 See [Ranges](Range.md) for detailed documentation on it.
 
+# Time routines
+
+These should be imported from [lyng.time](time.md). For example:
+
+    import lyng.time
+
+    val now = Instant()
+    val hourAgo = now - 1.hour
+
+See [more docs on time manipulation](time.md)
+
 # Comments
 
     // single line comment
@@ -1198,27 +1217,29 @@ Concatenation is a `+`: `"hello " + name` works as expected. No confusion.
 
 Typical set of String functions includes:
 
-| fun/prop           | description / notes                                        |
-|--------------------|------------------------------------------------------------|
-| lower()            | change case to unicode upper                               |
-| upper()            | change case to unicode lower                               |
+| fun/prop          | description / notes                                        |
+|-------------------|------------------------------------------------------------|
+| lower()           | change case to unicode upper                               |
+| upper()           | change case to unicode lower                               |
 | startsWith(prefix) | true if starts with a prefix                               |
-| endsWith(prefix)   | true if ends with a prefix                                 |
-| take(n)            | get a new string from up to n first characters             |
-| takeLast(n)        | get a new string from up to n last characters              |
-| drop(n)            | get a new string dropping n first chars, or empty string   |
-| dropLast(n)        | get a new string dropping n last chars, or empty string    |
-| size               | size in characters like `length` because String is [Array] |
-| (args...)          | sprintf-like formatting, see [string formatting]           |
-| [index]            | character at index                                         | 
-| [Range]            | substring at range                                         |
-| s1 + s2            | concatenation                                              |
-| s1 += s2           | self-modifying concatenation                               |
-| toReal()           | attempts to parse string as a Real value                   |
-| toInt()            | parse string to Int value                                  |
-|                    |                                                            |
-
+| endsWith(prefix)  | true if ends with a prefix                                 |
+| take(n)           | get a new string from up to n first characters             |
+| takeLast(n)       | get a new string from up to n last characters              |
+| drop(n)           | get a new string dropping n first chars, or empty string   |
+| dropLast(n)       | get a new string dropping n last chars, or empty string    |
+| size              | size in characters like `length` because String is [Array] |
+| (args...)         | sprintf-like formatting, see [string formatting]           |
+| [index]           | character at index                                         | 
+| [Range]           | substring at range                                         |
+| s1 + s2           | concatenation                                              |
+| s1 += s2          | self-modifying concatenation                               |
+| toReal()          | attempts to parse string as a Real value                   |
+| toInt()           | parse string to Int value                                  |
+| characters()      | create [List] of characters (1)                            |
+| encodeUtf8()      | returns [Buffer] with characters encoded to utf8           |
 
+(1)
+: List is mutable therefore a new copy is created on each call.
 
 
 
@@ -1256,4 +1277,5 @@ See [math functions](math.md). Other general purpose functions are:
 [String]: String.md
 [string formatting]: https://github.com/sergeych/mp_stools?tab=readme-ov-file#sprintf-syntax-summary
 [Set]: Set.md
-[Map]: Map.md
+[Map]: Map.md
+[Buffer]: Buffer.md

gradle/libs.versions.toml

@@ -8,6 +8,7 @@ kotlinx-coroutines = "1.10.1"
 mp_bintools = "0.1.12"
 firebaseCrashlyticsBuildtools = "3.0.3"
 okioVersion = "3.10.2"
+compiler = "3.2.0-alpha11"
 
 [libraries]
 clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" }
@@ -19,6 +20,7 @@ mp_bintools = { module = "net.sergeych:mp_bintools", version.ref = "mp_bintools"
 firebase-crashlytics-buildtools = { group = "com.google.firebase", name = "firebase-crashlytics-buildtools", version.ref = "firebaseCrashlyticsBuildtools" }
 okio = { module = "com.squareup.okio:okio", version.ref = "okioVersion" }
 okio-fakefilesystem = { module = "com.squareup.okio:okio-fakefilesystem", version.ref = "okioVersion" }
+compiler = { group = "androidx.databinding", name = "compiler", version.ref = "compiler" }
 
 [plugins]
 androidLibrary = { id = "com.android.library", version.ref = "agp" }

lyng/src/commonMain/kotlin/Common.kt

@@ -1,12 +1,19 @@
 package net.sergeych
 
 import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.core.Context
+import com.github.ajalt.clikt.core.main
 import com.github.ajalt.clikt.parameters.arguments.argument
 import com.github.ajalt.clikt.parameters.arguments.multiple
 import com.github.ajalt.clikt.parameters.arguments.optional
 import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
-import net.sergeych.lyng.*
+import kotlinx.coroutines.runBlocking
+import net.sergeych.lyng.LyngVersion
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.ScriptError
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.obj.*
 import okio.FileSystem
 import okio.Path.Companion.toPath
 import okio.SYSTEM
@@ -41,7 +48,23 @@ val baseScope = Scope().apply {
 //    }
 }
 
-class Lyng(val launcher: (suspend () -> Unit) -> Unit) : CliktCommand() {
+fun runMain(args: Array<String>) {
+    if(args.isNotEmpty()) {
+        if( args.size >= 2 && args[0] == "--" ) {
+            // -- -file.lyng <args>
+            executeFileWithArgs(args[1], args.drop(2))
+            return
+        } else if( args[0][0] != '-') {
+            // file.lyng <args>
+            executeFileWithArgs(args[0], args.drop(1))
+            return
+        }
+    }
+    // normal processing
+    Lyng { runBlocking { it() } }.main(args)
+}
+
+private class Lyng(val launcher: (suspend () -> Unit) -> Unit) : CliktCommand() {
 
     override val printHelpOnEmptyArgs = true
 
@@ -55,7 +78,7 @@ class Lyng(val launcher: (suspend () -> Unit) -> Unit) : CliktCommand() {
 
     val args by argument(help = "arguments for script").multiple()
 
-    override fun help(context: com.github.ajalt.clikt.core.Context): String =
+    override fun help(context: Context): String =
         """
             The Lyng script language interpreter, language version is $LyngVersion.
             
@@ -106,6 +129,13 @@ class Lyng(val launcher: (suspend () -> Unit) -> Unit) : CliktCommand() {
     }
 }
 
+fun executeFileWithArgs(fileName: String, args: List<String>) {
+    runBlocking {
+        baseScope.addConst("ARGV", ObjList(args.map { ObjString(it) }.toMutableList()))
+        executeFile(fileName)
+    }
+}
+
 suspend fun executeFile(fileName: String) {
     var text = FileSystem.SYSTEM.source(fileName.toPath()).use { fileSource ->
         fileSource.buffer().use { bs ->
@@ -118,7 +148,7 @@ suspend fun executeFile(fileName: String) {
         text = text.substring(pos + 1)
     }
     processErrors {
-        Compiler.compile(Source(fileName, text)).execute(baseScope)
+        baseScope.eval(Source(fileName, text))
     }
 }
 

lyng/src/jvmMain/kotlin/net/sergeych/lyng_cli/Main.kt

@@ -1,9 +1,7 @@
 package net.sergeych.lyng_cli
 
-import com.github.ajalt.clikt.core.main
-import kotlinx.coroutines.runBlocking
-import net.sergeych.Lyng
+import net.sergeych.runMain
 
 fun main(args: Array<String>) {
-    Lyng({ runBlocking { it() } }).main(args)
+    runMain(args)
 }

lyng/src/linuxX64Main/kotlin/Main.kt

@@ -1,7 +1,5 @@
-import com.github.ajalt.clikt.core.main
-import kotlinx.coroutines.runBlocking
-import net.sergeych.Lyng
+import net.sergeych.runMain
 
 fun main(args: Array<String>) {
-    Lyng( { runBlocking { it() } }).main(args)
+    runMain(args)
 }

lynglib/build.gradle.kts

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 group = "net.sergeych"
-version = "0.7.0-SNAPSHOT"
+version = "0.7.4-SNAPSHOT"
 
 buildscript {
     repositories {
@@ -20,7 +20,7 @@ plugins {
     alias(libs.plugins.kotlinMultiplatform)
     alias(libs.plugins.androidLibrary)
 //    alias(libs.plugins.vanniktech.mavenPublish)
-    kotlin("plugin.serialization") version "2.1.20"
+    kotlin("plugin.serialization") version "2.2.0"
     id("com.codingfeline.buildkonfig") version "0.17.1"
     `maven-publish`
 }
@@ -99,6 +99,7 @@ android {
 }
 dependencies {
     implementation(libs.firebase.crashlytics.buildtools)
+    implementation(libs.compiler)
 }
 
 publishing {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/AppliedScope.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.ObjRecord
+
 /**
  * Special version of the [Scope] used to `apply` new this object to
  * _parent context property.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjList
+
 /**
  * List of argument declarations in the __definition__ of the lambda, class constructor,
  * function, etc. It is created by [Compiler.parseArgsDeclaration]
@@ -54,8 +57,8 @@ data class ArgsDeclaration(val params: List<Item>, val endTokenType: Token.Type)
                     i < callArgs.size -> callArgs[i]
                     a.defaultValue != null -> a.defaultValue.execute(scope)
                     else -> {
-                        println("callArgs: ${callArgs.joinToString()}")
-                        println("tailBlockMode: ${arguments.tailBlockMode}")
+//                        println("callArgs: ${callArgs.joinToString()}")
+//                        println("tailBlockMode: ${arguments.tailBlockMode}")
                         scope.raiseIllegalArgument("too few arguments for the call")
                     }
                 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Arguments.kt

@@ -1,5 +1,9 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjIterable
+import net.sergeych.lyng.obj.ObjList
+
 data class ParsedArgument(val value: Statement, val pos: Pos, val isSplat: Boolean = false)
 
 suspend fun Collection<ParsedArgument>.toArguments(scope: Scope, tailBlockMode: Boolean): Arguments {
@@ -26,7 +30,7 @@ suspend fun Collection<ParsedArgument>.toArguments(scope: Scope, tailBlockMode:
     return Arguments(list,tailBlockMode)
 }
 
-data class Arguments(val list: List<Obj>,val tailBlockMode: Boolean = false) : List<Obj> by list {
+data class Arguments(val list: List<Obj>, val tailBlockMode: Boolean = false) : List<Obj> by list {
 
     constructor(vararg values: Obj) : this(values.toList())
 
@@ -42,6 +46,9 @@ data class Arguments(val list: List<Obj>,val tailBlockMode: Boolean = false) : L
         return list.map { it.toKotlin(scope) }
     }
 
+    fun inspect(): String = list.joinToString(", ") { it.inspect() }
+
+
     companion object {
         val EMPTY = Arguments(emptyList())
         fun from(values: Collection<Obj>) = Arguments(values.toList())

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1,14 +1,18 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.ImportProvider
+
 /**
  * The LYNG compiler.
  */
 class Compiler(
     val cc: CompilerContext,
-    val pacman: Pacman = Pacman.emptyAllowAll,
+    val importManager: ImportProvider,
     @Suppress("UNUSED_PARAMETER")
     settings: Settings = Settings()
 ) {
+
     var packageName: String? = null
 
     class Settings
@@ -20,6 +24,10 @@ class Compiler(
         // package level declarations
         do {
             val t = cc.current()
+            if (t.type == Token.Type.NEWLINE || t.type == Token.Type.SINLGE_LINE_COMMENT || t.type == Token.Type.MULTILINE_COMMENT) {
+                cc.next()
+                continue
+            }
             if (t.type == Token.Type.ID) {
                 when (t.value) {
                     "package" -> {
@@ -33,15 +41,17 @@ class Compiler(
                         packageName = name
                         continue
                     }
+
                     "import" -> {
                         cc.next()
                         val pos = cc.currentPos()
                         val name = loadQualifiedName()
-                        pacman.prepareImport(pos, name, null)
+                        val module = importManager.prepareImport(pos, name, null)
                         statements += statement {
-                            pacman.performImport(this,name,null)
+                            module.importInto(this, null)
                             ObjVoid
                         }
+                        continue
                     }
                 }
             }
@@ -64,6 +74,7 @@ class Compiler(
                 t = cc.next()
             }
         }
+        cc.previous()
         return result.toString()
     }
 
@@ -266,9 +277,7 @@ class Compiler(
                             if (x == ObjNull && isOptional) ObjNull.asReadonly
                             else x.getAt(cxt, i).asMutable
                         }) { cxt, newValue ->
-                            val i = (index.execute(cxt) as? ObjInt)?.value?.toInt()
-                                ?: cxt.raiseError("index must be integer")
-                            left.getter(cxt).value.putAt(cxt, i, newValue)
+                            left.getter(cxt).value.putAt(cxt, index.execute(cxt), newValue)
                         }
                     } ?: run {
                         // array literal
@@ -338,17 +347,26 @@ class Compiler(
                         left.setter(startPos)
                         operand = Accessor { cxt ->
                             val x = left.getter(cxt)
-                            if (x.isMutable)
-                                x.value.getAndIncrement(cxt).asReadonly
-                            else cxt.raiseError("Cannot increment immutable value")
+                            if (x.isMutable) {
+                                if (x.value.isConst) {
+                                    x.value.plus(cxt, ObjInt.One).also {
+                                        left.setter(startPos)(cxt, it)
+                                    }.asReadonly
+                                } else
+                                    x.value.getAndIncrement(cxt).asReadonly
+                            } else cxt.raiseError("Cannot increment immutable value")
                         }
                     } ?: run {
                         // no lvalue means pre-increment, expression to increment follows
-                        val next = parseAccessor() ?: throw ScriptError(t.pos, "Expecting expression")
+                        val next = parseTerm() ?: throw ScriptError(t.pos, "Expecting expression")
                         operand = Accessor { ctx ->
-                            next.getter(ctx).also {
+                            val x = next.getter(ctx).also {
                                 if (!it.isMutable) ctx.raiseError("Cannot increment immutable value")
-                            }.value.incrementAndGet(ctx).asReadonly
+                            }.value
+                            if (x.isConst) {
+                                next.setter(startPos)(ctx, x.plus(ctx, ObjInt.One))
+                                x.asReadonly
+                            } else x.incrementAndGet(ctx).asReadonly
                         }
                     }
                 }
@@ -358,18 +376,28 @@ class Compiler(
                     operand?.let { left ->
                         // post decrement
                         left.setter(startPos)
-                        operand = Accessor { ctx ->
-                            left.getter(ctx).also {
-                                if (!it.isMutable) ctx.raiseError("Cannot decrement immutable value")
-                            }.value.getAndDecrement(ctx).asReadonly
+                        operand = Accessor { cxt ->
+                            val x = left.getter(cxt)
+                            if (!x.isMutable) cxt.raiseError("Cannot decrement immutable value")
+                            if (x.value.isConst) {
+                                x.value.minus(cxt, ObjInt.One).also {
+                                    left.setter(startPos)(cxt, it)
+                                }.asReadonly
+                            } else
+                                x.value.getAndDecrement(cxt).asReadonly
                         }
                     } ?: run {
                         // no lvalue means pre-decrement, expression to decrement follows
-                        val next = parseAccessor() ?: throw ScriptError(t.pos, "Expecting expression")
-                        operand = Accessor { ctx ->
-                            next.getter(ctx).also {
-                                if (!it.isMutable) ctx.raiseError("Cannot decrement immutable value")
-                            }.value.decrementAndGet(ctx).asReadonly
+                        val next = parseTerm() ?: throw ScriptError(t.pos, "Expecting expression")
+                        operand = Accessor { cxt ->
+                            val x = next.getter(cxt)
+                            if (!x.isMutable) cxt.raiseError("Cannot decrement immutable value")
+                            if (x.value.isConst) {
+                                x.value.minus(cxt, ObjInt.One).also {
+                                    next.setter(startPos)(cxt, it)
+                                }.asReadonly
+                            } else
+                                x.value.decrementAndGet(cxt).asReadonly
                         }
                     }
                 }
@@ -687,7 +715,7 @@ class Compiler(
         }
     }
 
-    private fun parseAccessor(): Accessor? {
+    private suspend fun parseAccessor(): Accessor? {
         // could be: literal
         val t = cc.next()
         return when (t.type) {
@@ -709,8 +737,14 @@ class Compiler(
             }
 
             Token.Type.MINUS -> {
-                val n = parseNumber(false)
-                Accessor { n.asReadonly }
+                parseNumberOrNull(false)?.let { n ->
+                    Accessor { n.asReadonly }
+                } ?: run {
+                    val n = parseTerm() ?: throw ScriptError(t.pos, "Expecting expression after unary minus")
+                    Accessor {
+                        n.getter.invoke(it).value.negate(it).asReadonly
+                    }
+                }
             }
 
             Token.Type.ID -> {
@@ -741,11 +775,12 @@ class Compiler(
         }
     }
 
-    private fun parseNumber(isPlus: Boolean): Obj {
+    private fun parseNumberOrNull(isPlus: Boolean): Obj? {
+        val pos = cc.savePos()
         val t = cc.next()
         return when (t.type) {
             Token.Type.INT, Token.Type.HEX -> {
-                val n = t.value.toLong(if (t.type == Token.Type.HEX) 16 else 10)
+                val n = t.value.replace("_", "").toLong(if (t.type == Token.Type.HEX) 16 else 10)
                 if (isPlus) ObjInt(n) else ObjInt(-n)
             }
 
@@ -755,11 +790,16 @@ class Compiler(
             }
 
             else -> {
-                throw ScriptError(t.pos, "expected number")
+                cc.restorePos(pos)
+                null
             }
         }
     }
 
+    private fun parseNumber(isPlus: Boolean): Obj {
+        return parseNumberOrNull(isPlus) ?: throw ScriptError(cc.currentPos(), "Expecting number")
+    }
+
     /**
      * Parse keyword-starting statement.
      * @return parsed statement or null if, for example. [id] is not among keywords
@@ -1084,6 +1124,7 @@ class Compiler(
         // inheritance must alter this code:
         val newClass = ObjClass(className).apply {
             instanceConstructor = constructorCode
+            constructorMeta = constructorArgsDeclaration
         }
 
         return statement {
@@ -1612,15 +1653,8 @@ class Compiler(
 
     companion object {
 
-        suspend fun compile(source: Source,pacman: Pacman = Pacman.emptyAllowAll): Script {
-            return Compiler(CompilerContext(parseLyng(source)),pacman).parseScript()
-        }
-
-        suspend fun compilePackage(source: Source): Pair<String, Script> {
-            val c = Compiler(CompilerContext(parseLyng(source)))
-            val script = c.parseScript()
-            if (c.packageName == null) throw ScriptError(source.startPos, "package not set")
-            return c.packageName!! to script
+        suspend fun compile(source: Source, importManager: ImportProvider): Script {
+            return Compiler(CompilerContext(parseLyng(source)), importManager).parseScript()
         }
 
         private var lastPriority = 0
@@ -1743,7 +1777,7 @@ class Compiler(
             allOps.filter { it.priority == l }.associateBy { it.tokenType }
         }
 
-        suspend fun compile(code: String): Script = compile(Source("<eval>", code))
+        suspend fun compile(code: String): Script = compile(Source("<eval>", code), Script.defaultImportManager)
 
         /**
          * The keywords that stop processing of expression term

lynglib/src/commonMain/kotlin/net/sergeych/lyng/CompilerContext.kt

@@ -164,7 +164,9 @@ class CompilerContext(val tokens: List<Token>) {
      * Note that [Token.Type.EOF] is not considered a whitespace token.
      */
     fun skipWsTokens(): Token {
-        while( current().type in wstokens ) next()
+        while( current().type in wstokens ) {
+            next()
+        }
         return next()
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Importer.kt

@@ -1,130 +0,0 @@
-package net.sergeych.lyng
-
-import kotlinx.coroutines.Deferred
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
-import net.sergeych.mp_tools.globalDefer
-
-/**
- * Package manager. Chained manager, too simple. Override [createModuleScope] to return either
- * valid [ModuleScope] or call [parent] - or return null.
- */
-abstract class Pacman(
-    val parent: Pacman? = null,
-    val rootScope: Scope = parent!!.rootScope,
-    val securityManager: SecurityManager = parent!!.securityManager
-) {
-    private val opScopes = Mutex()
-
-    private val cachedScopes = mutableMapOf<String, Scope>()
-
-    /**
-     * Create a new module scope if this pacman can import the module, return null otherwise so
-     * the manager can decide what to do
-     */
-    abstract suspend fun createModuleScope(name: String): ModuleScope?
-
-    suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?) {
-        if (!securityManager.canImportModule(name))
-            throw ImportException(pos, "Module $name is not allowed")
-        symbols?.keys?.forEach {
-            if (!securityManager.canImportSymbol(name, it)) throw ImportException(
-                pos,
-                "Symbol $name.$it is not allowed"
-            )
-        }
-        // if we can import the module, cache it, or go further
-        opScopes.withLock {
-            cachedScopes[name] ?: createModuleScope(name)?.let { cachedScopes[name] = it }
-        } ?: parent?.prepareImport(pos, name, symbols) ?: throw ImportException(pos, "Module $name is not found")
-    }
-
-    suspend fun performImport(scope: Scope, name: String, symbols: Map<String, String>?) {
-        val module = opScopes.withLock { cachedScopes[name] }
-            ?: scope.raiseSymbolNotFound("module $name not found")
-        val symbolsToImport = symbols?.keys?.toMutableSet()
-        for ((symbol, record) in module.objects) {
-            if (record.visibility.isPublic) {
-                println("import $name: $symbol: $record")
-                val newName = symbols?.let { ss: Map<String, String> ->
-                    ss[symbol]
-                        ?.also { symbolsToImport!!.remove(it) }
-                        ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
-                } ?: symbol
-                println("import $name.$symbol as $newName")
-                if (newName in scope.objects)
-                    scope.raiseError("symbol $newName already exists, redefinition on import is not allowed")
-                scope.objects[newName] = record
-            }
-        }
-        if (!symbolsToImport.isNullOrEmpty())
-            scope.raiseSymbolNotFound("symbols $name.{$symbolsToImport} are.were not found")
-    }
-
-    companion object {
-        val emptyAllowAll = object : Pacman(rootScope = Script.defaultScope, securityManager = SecurityManager.allowAll) {
-            override suspend fun createModuleScope(name: String): ModuleScope? {
-                return null
-            }
-
-        }
-    }
-
-}
-
-/**
- * Module scope supports importing and contains the [pacman]; it should be the same
- * used in [Compiler];
- */
-class ModuleScope(
-    val pacman: Pacman,
-    pos: Pos = Pos.builtIn,
-    val packageName: String
-) : Scope(pacman.rootScope, Arguments.EMPTY, pos) {
-
-    constructor(pacman: Pacman,source: Source) : this(pacman, source.startPos, source.fileName)
-
-    override suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>?) {
-        pacman.prepareImport(pos, name, symbols)
-    }
-
-    /**
-     * Import symbols into the scope. It _is called_ after the module is imported by [checkImport].
-     * If [checkImport] was not called, the symbols will not be imported with exception as module is not found.
-     */
-    override suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>?) {
-        pacman.performImport(scope, name, symbols)
-    }
-
-    val packageNameObj by lazy { ObjString(packageName).asReadonly}
-
-    override fun get(name: String): ObjRecord? {
-        return if( name == "__PACKAGE__")
-            packageNameObj
-        else
-            super.get(name)
-    }
-}
-
-
-class InlineSourcesPacman(pacman: Pacman,val sources: List<Source>) : Pacman(pacman) {
-
-    val modules: Deferred<Map<String,Deferred<ModuleScope>>> = globalDefer {
-        val result = mutableMapOf<String, Deferred<ModuleScope>>()
-        for (source in sources) {
-            // retrieve the module name and script for deferred execution:
-            val (name, script) = Compiler.compilePackage(source)
-            // scope is created used pacman's root scope:
-            val scope = ModuleScope(this@InlineSourcesPacman, source.startPos, name)
-            // we execute scripts in parallel which allow cross-imports to some extent:
-            result[name] = globalDefer { script.execute(scope); scope }
-        }
-        result
-    }
-
-    override suspend fun createModuleScope(name: String): ModuleScope? =
-        modules.await()[name]?.await()
-
-}
-
-

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ListEntry.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Accessor
+
 sealed class ListEntry {
     data class Element(val accessor: Accessor) : ListEntry()
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/LoopBreakContinueException.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjVoid
+
 class LoopBreakContinueException(
     val doContinue: Boolean,
     val result: Obj = ObjVoid,

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt

@@ -0,0 +1,61 @@
+package net.sergeych.lyng
+
+import net.sergeych.lyng.obj.ObjRecord
+import net.sergeych.lyng.obj.ObjString
+import net.sergeych.lyng.pacman.ImportProvider
+
+/**
+ * Module scope supports importing and contains the [importProvider]; it should be the same
+ * used in [Compiler];
+ */
+class ModuleScope(
+    var importProvider: ImportProvider,
+    pos: Pos = Pos.builtIn,
+    override val packageName: String
+) : Scope(importProvider.rootScope, Arguments.EMPTY, pos) {
+
+    constructor(importProvider: ImportProvider, source: Source) : this(importProvider, source.startPos, source.fileName)
+
+    /**
+     * Import symbols into the scope. It _is called_ after the module is imported by [ImportProvider.prepareImport]
+     * which checks symbol availability and accessibility prior to execution.
+     * @param scope where to copy symbols from this module
+     * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
+     */
+    override suspend fun importInto(scope: Scope, symbols: Map<String, String>?) {
+        val symbolsToImport = symbols?.keys?.toMutableSet()
+        for ((symbol, record) in this.objects) {
+            if (record.visibility.isPublic) {
+                val newName = symbols?.let { ss: Map<String, String> ->
+                    ss[symbol]
+                        ?.also { symbolsToImport!!.remove(it) }
+                        ?: scope.raiseError("internal error: symbol $symbol not found though the module is cached")
+                } ?: symbol
+                val existing = scope.objects[newName]
+                if (existing != null ) {
+                    if (existing.importedFrom != record.importedFrom)
+                        scope.raiseError("symbol ${existing.importedFrom?.packageName}.$newName already exists, redefinition on import is not allowed")
+                    // already imported
+                }
+                else {
+                    // when importing records, we keep track of its package (not otherwise needed)
+                    if (record.importedFrom == null) record.importedFrom = this
+                    scope.objects[newName] = record
+                }
+            }
+        }
+        if (!symbolsToImport.isNullOrEmpty())
+            scope.raiseSymbolNotFound("symbols $packageName.{$symbolsToImport} are.were not found")
+    }
+
+    val packageNameObj by lazy { ObjString(packageName).asReadonly }
+
+    override fun get(name: String): ObjRecord? {
+        return if (name == "__PACKAGE__")
+            packageNameObj
+        else
+            super.get(name)
+    }
+}
+
+

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ObjBool.kt

@@ -1,31 +0,0 @@
-package net.sergeych.lyng
-
-data class ObjBool(val value: Boolean) : Obj() {
-    override val asStr by lazy { ObjString(value.toString()) }
-
-    override suspend fun compareTo(scope: Scope, other: Obj): Int {
-        if (other !is ObjBool) return -2
-        return value.compareTo(other.value)
-    }
-
-    override fun toString(): String = value.toString()
-
-    override val objClass: ObjClass = type
-
-    override suspend fun logicalNot(scope: Scope): Obj = ObjBool(!value)
-
-    override suspend fun logicalAnd(scope: Scope, other: Obj): Obj = ObjBool(value && other.toBool())
-
-    override suspend fun logicalOr(scope: Scope, other: Obj): Obj = ObjBool(value || other.toBool())
-
-    override suspend fun toKotlin(scope: Scope): Any {
-        return value
-    }
-
-    companion object {
-        val type = ObjClass("Bool")
-    }
-}
-
-val ObjTrue = ObjBool(true)
-val ObjFalse = ObjBool(false)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ObjIterator.kt

@@ -1,3 +0,0 @@
-package net.sergeych.lyng
-
-val ObjIterator by lazy { ObjClass("Iterator") }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt

@@ -109,7 +109,7 @@ private class Parser(fromPos: Pos) {
             '/' -> when (currentChar) {
                 '/' -> {
                     pos.advance()
-                    Token(loadToEnd().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
+                    Token(loadToEndOfLine().trim(), from, Token.Type.SINLGE_LINE_COMMENT)
                 }
 
                 '*' -> {
@@ -250,7 +250,7 @@ private class Parser(fromPos: Pos) {
 
             in digitsSet -> {
                 pos.back()
-                decodeNumber(loadChars(digits), from)
+                decodeNumber(loadChars { it in digitsSet || it == '_'}, from)
             }
 
             '\'' -> {
@@ -445,7 +445,7 @@ private class Parser(fromPos: Pos) {
         }
     }
 
-    private fun loadToEnd(): String {
+    private fun loadToEndOfLine(): String {
         val result = StringBuilder()
         val l = pos.line
         do {
@@ -479,4 +479,10 @@ private class Parser(fromPos: Pos) {
         return null
     }
 
+    init {
+        // skip shebang
+        if( pos.readFragment("#!") )
+            loadToEndOfLine()
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -1,13 +1,22 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.ImportManager
+import net.sergeych.lyng.pacman.ImportProvider
+
 /**
  * Scope is where local variables and methods are stored. Scope is also a parent scope for other scopes.
  * Each block usually creates a scope. Accessing Lyng closures usually is done via a scope.
  *
- * There are special types of scopes:
+ * To create default scope, use default `Scope()` constructor, it will create a scope with a parent
+ * module scope with default [ImportManager], you can access with [currentImportProvider] as needed.
  *
- * - [Script.defaultScope] - root scope for a script, safe one
- * - [AppliedScope] - scope used to apply a closure to some thisObj scope
+ * If you want to create [ModuleScope] by hand, try [currentImportProvider] and [ImportManager.newModule],
+ * or [ImportManager.newModuleAt].
+ *
+ *  There are special types of scopes:
+ *
+ *  - [AppliedScope] - scope used to apply a closure to some thisObj scope
  */
 open class Scope(
     val parent: Scope?,
@@ -16,11 +25,13 @@ open class Scope(
     var thisObj: Obj = ObjVoid,
     var skipScopeCreation: Boolean = false,
 ) {
+    open val packageName: String = "<anonymous package>"
+
     constructor(
         args: Arguments = Arguments.EMPTY,
         pos: Pos = Pos.builtIn,
     )
-            : this(Script.defaultScope, args, pos)
+            : this(Script.defaultImportManager.copy().newModuleAt(pos), args, pos)
 
     fun raiseNotImplemented(what: String = "operation"): Nothing = raiseError("$what is not implemented")
 
@@ -35,6 +46,10 @@ open class Scope(
     fun raiseIllegalArgument(message: String = "Illegal argument error"): Nothing =
         raiseError(ObjIllegalArgumentException(this, message))
 
+    @Suppress("unused")
+    fun raiseIllegalState(message: String = "Illegal argument error"): Nothing =
+        raiseError(ObjIllegalStateException(this, message))
+
     @Suppress("unused")
     fun raiseNoSuchElement(message: String = "No such element"): Nothing =
         raiseError(ObjIllegalArgumentException(this, message))
@@ -71,6 +86,11 @@ open class Scope(
         }
     }
 
+    fun requireNoArgs() {
+        if( args.list.isNotEmpty())
+            raiseError("This function does not accept any arguments")
+    }
+
     inline fun <reified T : Obj> thisAs(): T = (thisObj as? T)
         ?: raiseClassCastError("Cannot cast ${thisObj.objClass.className} to ${T::class.simpleName}")
 
@@ -132,21 +152,46 @@ open class Scope(
     fun addConst(name: String, value: Obj) = addItem(name, false, value)
 
     suspend fun eval(code: String): Obj =
-        Compiler.compile(code.toSource()).execute(this)
+        Compiler.compile(code.toSource(), currentImportProvider).execute(this)
 
     suspend fun eval(source: Source): Obj =
         Compiler.compile(
             source,
-            (this as? ModuleScope)?.pacman?.also { println("pacman found: $pacman")} ?: Pacman.emptyAllowAll
-            ).execute(this)
+            currentImportProvider
+        ).execute(this)
 
     fun containsLocal(name: String): Boolean = name in objects
 
-    open suspend fun checkImport(pos: Pos, name: String, symbols: Map<String, String>? = null) {
-        throw ImportException(pos, "Import is not allowed here: $name")
+    /**
+     * Some scopes can be imported into other scopes, like [ModuleScope]. Those must correctly implement this method.
+     * @param scope where to copy symbols from this module
+     * @param symbols symbols to import, ir present, only symbols keys will be imported renamed to corresponding values
+     */
+    open suspend fun importInto(scope: Scope, symbols: Map<String, String>? = null) {
+        scope.raiseError(ObjIllegalOperationException(scope, "Import is not allowed here: import $packageName"))
     }
 
-    open suspend fun importInto(scope: Scope, name: String, symbols: Map<String, String>? = null) {
-        scope.raiseError(ObjIllegalOperationException(scope,"Import is not allowed here: import $name"))
+    /**
+     * Find a first [ImportManager] in this Scope hierarchy. Normally there should be one. Found instance is cached.
+     *
+     * Use it to register your package sources, see [ImportManager] features.
+     *
+     * @throws IllegalStateException if there is no such manager (if you create some specific scope with no manager,
+     *      then you knew what you did)
+     */
+    val currentImportProvider: ImportProvider by lazy {
+        if (this is ModuleScope)
+            importProvider.getActualProvider()
+        else
+            parent?.currentImportProvider ?: throw IllegalStateException("this scope has no manager in the chain")
+    }
+
+    val importManager by lazy { (currentImportProvider as? ImportManager)
+        ?: throw IllegalStateException("this scope has no manager in the chain (provided $currentImportProvider") }
+
+    companion object {
+
+        fun new(): Scope =
+            Script.defaultImportManager.copy().newModuleAt(Pos.builtIn)
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt

@@ -1,6 +1,9 @@
 package net.sergeych.lyng
 
 import kotlinx.coroutines.delay
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.ImportManager
+import net.sergeych.lynon.ObjLynonClass
 import kotlin.math.*
 
 class Script(
@@ -17,10 +20,11 @@ class Script(
         return lastResult
     }
 
-    suspend fun execute() = execute(defaultScope.copy(pos = pos))
+    suspend fun execute() = execute(defaultImportManager.newModule())
 
     companion object {
-        val defaultScope: Scope = Scope().apply {
+
+        private val rootScope: Scope = Scope(null).apply {
             ObjException.addExceptionsToContext(this)
             addFn("println") {
                 for ((i, a) in args.withIndex()) {
@@ -112,7 +116,7 @@ class Script(
             }
             addFn( "abs" ) {
                 val x = args.firstAndOnly()
-                if( x is ObjInt ) ObjInt( x.value.absoluteValue ) else ObjReal( x.toDouble().absoluteValue )
+                if( x is ObjInt) ObjInt( x.value.absoluteValue ) else ObjReal( x.toDouble().absoluteValue )
             }
 
             addVoidFn("assert") {
@@ -170,8 +174,34 @@ class Script(
             getOrCreateNamespace("Math").apply {
                 addConst("PI", pi)
             }
+        }
 
+        val defaultImportManager: ImportManager by lazy {
+            ImportManager(rootScope, SecurityManager.allowAll).apply {
+                addPackage("lyng.buffer") {
+                    it.addConst("Buffer", ObjBuffer.type)
+                    it.addConst("MutableBuffer", ObjMutableBuffer.type)
+                }
+                addPackage("lyng.serialization") {
+                    it.addConst("Lynon", ObjLynonClass)
+                }
+                addPackage("lyng.time") {
+                    it.addConst("Instant", ObjInstant.type)
+                    it.addConst("Duration", ObjDuration.type)
+                    it.addFn("delay") {
+                        val a = args.firstAndOnly()
+                        when(a) {
+                            is ObjInt -> delay(a.value * 1000)
+                            is ObjReal -> delay((a.value * 1000).roundToLong())
+                            is ObjDuration -> delay(a.duration)
+                            else -> raiseIllegalArgument("Expected Duration, Int or Real, got ${a.inspect()}")
+                        }
+                        ObjVoid
+                    }
+                }
+            }
 
         }
+
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ScriptError.kt

@@ -2,6 +2,8 @@
 
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.ObjException
+
 open class ScriptError(val pos: Pos, val errorMessage: String, cause: Throwable? = null) : Exception(
     """
         $pos: Error: $errorMessage

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Source.kt

@@ -12,4 +12,15 @@ class Source(val fileName: String, text: String) {
     val startPos: Pos = Pos(this, 0, 0)
 
     fun posAt(line: Int, column: Int): Pos = Pos(this, line, column)
+
+    fun extractPackageName(): String {
+        for ((n,line) in lines.withIndex()) {
+            if( line.isBlank() || line.isEmpty() )
+                continue
+            if( line.startsWith("package ") )
+                return line.substring(8).trim()
+            else throw ScriptError(Pos(this, n, 0),"package declaration expected")
+        }
+        throw ScriptError(Pos(this, 0, 0),"package declaration expected")
+    }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Accessor.kt

@@ -0,0 +1,33 @@
+
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Compiler
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.ScriptError
+
+// avoid KDOC bug: keep it
+@Suppress("unused")
+typealias DocCompiler = Compiler
+/**
+ * When we need read-write access to an object in some abstract storage, we need Accessor,
+ * as in-site assigning is not always sufficient, in general case we need to replace the object
+ * in the storage.
+ *
+ * Note that assigning new value is more complex than just replacing the object, see how assignment
+ * operator is implemented in [Compiler.allOps].
+ */
+data class Accessor(
+    val getter: suspend (Scope) -> ObjRecord,
+    val setterOrNull: (suspend (Scope, Obj) -> Unit)?
+) {
+    /**
+     * Simplified constructor for immutable stores.
+     */
+    constructor(getter: suspend (Scope) -> ObjRecord) : this(getter, null)
+
+    /**
+     * Get the setter or throw.
+     */
+    fun setter(pos: Pos) = setterOrNull ?: throw ScriptError(pos, "can't assign value")
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt

@@ -1,48 +1,26 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import net.sergeych.bintools.encodeToHex
+import net.sergeych.lyng.*
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import net.sergeych.synctools.ProtectedOp
 import net.sergeych.synctools.withLock
 import kotlin.contracts.ExperimentalContracts
 
-/**
- * Record to store object with access rules, e.g. [isMutable] and access level [visibility].
- */
-data class ObjRecord(
-    var value: Obj,
-    val isMutable: Boolean,
-    val visibility: Visibility = Visibility.Public
-)
-
-/**
- * When we need read-write access to an object in some abstract storage, we need Accessor,
- * as in-site assigning is not always sufficient, in general case we need to replace the object
- * in the storage.
- *
- * Note that assigning new value is more complex than just replacing the object, see how assignment
- * operator is implemented in [Compiler.allOps].
- */
-data class Accessor(
-    val getter: suspend (Scope) -> ObjRecord,
-    val setterOrNull: (suspend (Scope, Obj) -> Unit)?
-) {
-    /**
-     * Simplified constructor for immutable stores.
-     */
-    constructor(getter: suspend (Scope) -> ObjRecord) : this(getter, null)
-
-    /**
-     * Get the setter or throw.
-     */
-    fun setter(pos: Pos) = setterOrNull ?: throw ScriptError(pos, "can't assign value")
-}
-
 open class Obj {
 
+    open val isConst: Boolean = false
+
+    fun ensureNotConst(scope: Scope) {
+        if (isConst) scope.raiseError("can't assign to constant")
+    }
+
     val isNull by lazy { this === ObjNull }
 
     var isFrozen: Boolean = false
@@ -125,6 +103,10 @@ open class Obj {
         scope.raiseNotImplemented()
     }
 
+    open suspend fun negate(scope: Scope): Obj {
+        scope.raiseNotImplemented()
+    }
+
     open suspend fun mul(scope: Scope, other: Obj): Obj {
         scope.raiseNotImplemented()
     }
@@ -220,7 +202,7 @@ open class Obj {
 
     suspend fun getAt(scope: Scope, index: Int): Obj = getAt(scope, ObjInt(index.toLong()))
 
-    open suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
+    open suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
         scope.raiseNotImplemented("indexing")
     }
 
@@ -256,31 +238,47 @@ open class Obj {
     val asReadonly: ObjRecord by lazy { ObjRecord(this, false) }
     val asMutable: ObjRecord by lazy { ObjRecord(this, true) }
 
+    open suspend fun lynonType(): LynonType = LynonType.Other
+
+    open suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        scope.raiseNotImplemented()
+    }
 
     companion object {
 
         val rootObjectType = ObjClass("Obj").apply {
-                addFn("toString") {
-                    thisObj.asStr
-                }
-                addFn("contains") {
-                    ObjBool(thisObj.contains(this, args.firstAndOnly()))
-                }
-                // utilities
-                addFn("let") {
-                    args.firstAndOnly().callOn(copy(Arguments(thisObj)))
-                }
-                addFn("apply") {
-                    val newContext = ( thisObj as? ObjInstance)?.instanceScope ?: this
-                    args.firstAndOnly()
-                        .callOn(newContext)
-                    thisObj
-                }
-                addFn("also") {
-                    args.firstAndOnly().callOn(copy(Arguments(thisObj)))
-                    thisObj
-                }
+            addFn("toString") {
+                thisObj.asStr
             }
+            addFn("contains") {
+                ObjBool(thisObj.contains(this, args.firstAndOnly()))
+            }
+            // utilities
+            addFn("let") {
+                args.firstAndOnly().callOn(copy(Arguments(thisObj)))
+            }
+            addFn("apply") {
+                val newContext = (thisObj as? ObjInstance)?.instanceScope ?: this
+                args.firstAndOnly()
+                    .callOn(newContext)
+                thisObj
+            }
+            addFn("also") {
+                args.firstAndOnly().callOn(copy(Arguments(thisObj)))
+                thisObj
+            }
+            addFn("getAt") {
+                requireExactCount(1)
+                thisObj.getAt(this, requiredArg<Obj>(0))
+            }
+            addFn("putAt") {
+                requireExactCount(2)
+                val newValue = args[1]
+                thisObj.putAt(this, requiredArg<Obj>(0), newValue)
+                newValue
+            }
+
+        }
 
 
         inline fun from(obj: Any?): Obj {
@@ -309,6 +307,8 @@ open class Obj {
     }
 }
 
+fun Double.toObj(): Obj = ObjReal(this)
+
 @Suppress("unused")
 inline fun <reified T> T.toObj(): Obj = Obj.from(this)
 
@@ -349,7 +349,7 @@ object ObjNull : Obj() {
         scope.raiseNPE()
     }
 
-    override suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
+    override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
         scope.raiseNPE()
     }
 
@@ -362,6 +362,26 @@ object ObjNull : Obj() {
     override suspend fun toKotlin(scope: Scope): Any? {
         return null
     }
+
+    override suspend fun lynonType(): LynonType {
+        return LynonType.Null
+    }
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        if (lynonType == null) {
+            encoder.putBit(0)
+        }
+    }
+
+    override val objClass: ObjClass by lazy {
+        object : ObjClass("Null") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj {
+                if (lynonType == LynonType.Null)
+                    return this@ObjNull
+                else
+                    scope.raiseIllegalState("can't deserialize null directly or with wrong type: ${lynonType}")
+            }
+        }
+    }
 }
 
 interface Numeric {
@@ -492,6 +512,9 @@ class ObjIndexOutOfBoundsException(scope: Scope, message: String = "index out of
 class ObjIllegalArgumentException(scope: Scope, message: String = "illegal argument") :
     ObjException("IllegalArgumentException", scope, message)
 
+class ObjIllegalStateException(scope: Scope, message: String = "illegal state") :
+    ObjException("IllegalStateException", scope, message)
+
 @Suppress("unused")
 class ObjNoSuchElementException(scope: Scope, message: String = "no such element") :
     ObjException("IllegalArgumentException", scope, message)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjArray.kt

@@ -1,4 +1,4 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 val ObjArray by lazy {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjArrayIterator.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjArrayIterator(val array: Obj) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBitBuffer.kt

@@ -0,0 +1,37 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.bintools.toDump
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.BitArray
+
+class ObjBitBuffer(val bitArray: BitArray) : Obj() {
+
+    override val objClass = type
+
+    override suspend fun getAt(scope: Scope, index: Obj): Obj {
+        return bitArray[index.toLong()].toObj()
+    }
+
+    companion object {
+        val type = object: ObjClass("BitBuffer", ObjArray) {
+
+        }.apply {
+            addFn("toBuffer") {
+                requireNoArgs()
+                ObjBuffer(thisAs<ObjBitBuffer>().bitArray.asUbyteArray())
+            }
+            addFn("toDump") {
+                requireNoArgs()
+                ObjString(
+                    thisAs<ObjBitBuffer>().bitArray.asUbyteArray().toDump()
+                )
+            }
+            addFn("size") {
+                thisAs<ObjBitBuffer>().bitArray.size.toObj()
+            }
+            addFn("sizeInBytes") {
+                ObjInt((thisAs<ObjBitBuffer>().bitArray.size + 7) shr 3)
+            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBool.kt

@@ -0,0 +1,59 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
+
+data class ObjBool(val value: Boolean) : Obj() {
+    override val asStr by lazy { ObjString(value.toString()) }
+
+    override suspend fun compareTo(scope: Scope, other: Obj): Int {
+        if (other !is ObjBool) return -2
+        return value.compareTo(other.value)
+    }
+
+    override fun hashCode(): Int {
+        return value.hashCode()
+    }
+
+    override fun toString(): String = value.toString()
+
+    override val objClass: ObjClass = type
+
+    override suspend fun logicalNot(scope: Scope): Obj = ObjBool(!value)
+
+    override suspend fun logicalAnd(scope: Scope, other: Obj): Obj = ObjBool(value && other.toBool())
+
+    override suspend fun logicalOr(scope: Scope, other: Obj): Obj = ObjBool(value || other.toBool())
+
+    override suspend fun toKotlin(scope: Scope): Any {
+        return value
+    }
+
+    override suspend fun lynonType(): LynonType = LynonType.Bool
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.encodeBoolean(value)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjBool
+
+        return value == other.value
+    }
+
+    companion object {
+        val type = object : ObjClass("Bool") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder,lynonType: LynonType?): Obj {
+                return ObjBool(decoder.unpackBoolean())
+            }
+        }
+    }
+}
+
+val ObjTrue = ObjBool(true)
+val ObjFalse = ObjBool(false)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt

@@ -0,0 +1,168 @@
+package net.sergeych.lyng.obj
+
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.toList
+import net.sergeych.bintools.encodeToHex
+import net.sergeych.bintools.toDump
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
+import kotlin.math.min
+
+open class ObjBuffer(val byteArray: UByteArray) : Obj() {
+
+    override val objClass: ObjClass = type
+
+    fun checkIndex(scope: Scope, index: Obj): Int {
+        if (index !is ObjInt)
+            scope.raiseIllegalArgument("index must be Int")
+        val i = index.value.toInt()
+        if (i < 0) scope.raiseIllegalArgument("index must be positive")
+        if (i >= byteArray.size)
+            scope.raiseIndexOutOfBounds("index $i is out of bounds 0..<${byteArray.size}")
+        return i
+    }
+
+    override suspend fun getAt(scope: Scope, index: Obj): Obj {
+        // notice: we create a copy if content, so we don't want it
+        // to be treated as modifiable, or putAt will not be called:
+        return if (index is ObjRange) {
+            val start: Int = index.startInt(scope)
+            val end: Int = index.exclusiveIntEnd(scope) ?: size
+            ObjBuffer(byteArray.sliceArray(start..<end))
+        } else ObjInt(byteArray[checkIndex(scope, index)].toLong(), true)
+    }
+
+    val size by byteArray::size
+
+    override fun hashCode(): Int {
+        return byteArray.hashCode()
+    }
+
+    override suspend fun compareTo(scope: Scope, other: Obj): Int {
+        if (other !is ObjBuffer) return super.compareTo(scope, other)
+        val limit = min(size, other.size)
+        for (i in 0..<limit) {
+            val own = byteArray[i]
+            val their = other.byteArray[i]
+            if (own < their) return -1
+            else if (own > their) return 1
+        }
+        if (size < other.size) return -1
+        if (size > other.size) return 1
+        return 0
+    }
+
+    override suspend fun plus(scope: Scope, other: Obj): Obj {
+        return if (other is ObjBuffer)
+            ObjBuffer(byteArray + other.byteArray)
+        else if (other.isInstanceOf(ObjIterable)) {
+            ObjBuffer(
+                byteArray + other.toFlow(scope).map { it.toLong().toUByte() }.toList().toTypedArray()
+                    .toUByteArray()
+            )
+        } else scope.raiseIllegalArgument("can't concatenate buffer with ${other.inspect()}")
+    }
+
+    override fun toString(): String {
+        return "Buffer(${byteArray.encodeToHex()})"
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjBuffer
+
+        return byteArray contentEquals other.byteArray
+    }
+
+    override suspend fun lynonType(): LynonType = LynonType.Buffer
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.encodeCached(byteArray) {
+            bout.compress(byteArray.asByteArray())
+        }
+    }
+
+    companion object {
+        private suspend fun createBufferFrom(scope: Scope, obj: Obj): ObjBuffer =
+            when (obj) {
+                is ObjBuffer -> ObjBuffer(obj.byteArray.copyOf())
+                is ObjInt -> {
+                    if (obj.value < 0)
+                        scope.raiseIllegalArgument("buffer size must be positive")
+                    val data = UByteArray(obj.value.toInt())
+                    ObjBuffer(data)
+                }
+
+                is ObjString -> ObjBuffer(obj.value.encodeToByteArray().asUByteArray())
+                else -> {
+                    if (obj.isInstanceOf(ObjIterable)) {
+                        ObjBuffer(
+                            obj.toFlow(scope).map { it.toLong().toUByte() }.toList().toTypedArray()
+                                .toUByteArray()
+                        )
+                    } else
+                        scope.raiseIllegalArgument(
+                            "can't construct buffer from ${obj.inspect()}"
+                        )
+                }
+            }
+
+        val type = object : ObjClass("Buffer", ObjArray) {
+            override suspend fun callOn(scope: Scope): Obj {
+                val args = scope.args.list
+                return when (args.size) {
+                    // empty buffer
+                    0 -> ObjBuffer(ubyteArrayOf())
+                    1 -> createBufferFrom(scope, args[0])
+                    else -> {
+                        // create buffer from array, each argument should be a byte then:
+                        val data = UByteArray(args.size)
+                        for ((i, b) in args.withIndex()) {
+                            val code = when (b) {
+                                is ObjChar -> b.value.code.toUByte()
+                                is ObjInt -> b.value.toUByte()
+                                else -> scope.raiseIllegalArgument(
+                                    "invalid byte value for buffer constructor at index $i: ${b.inspect()}"
+                                )
+                            }
+                            data[i] = code
+                        }
+                        ObjBuffer(data)
+                    }
+                }
+            }
+
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
+                ObjBuffer( decoder.decodeCached {
+                    decoder.decompress().asUByteArray()
+                })
+
+        }.apply {
+            createField("size",
+                statement {
+                    (thisObj as ObjBuffer).byteArray.size.toObj()
+                }
+            )
+            addFn("decodeUtf8") {
+                ObjString(
+                    thisAs<ObjBuffer>().byteArray.toByteArray().decodeToString()
+                )
+            }
+            addFn("toMutable") {
+                requireNoArgs()
+                ObjMutableBuffer(thisAs<ObjBuffer>().byteArray.copyOf())
+            }
+            addFn("toDump") {
+                requireNoArgs()
+                ObjString(
+                    thisAs<ObjBuffer>().byteArray.toByteArray().toDump()
+                )
+            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjChar.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjChar(val value: Char): Obj() {
 
@@ -11,6 +13,19 @@ class ObjChar(val value: Char): Obj() {
 
     override fun inspect(): String = "'$value'"
 
+    override fun hashCode(): Int {
+        return value.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjChar
+
+        return value == other.value
+    }
+
     companion object {
         val type = ObjClass("Char").apply {
             addFn("code") { ObjInt(thisAs<ObjChar>().value.code.toLong()) }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjClass.kt

@@ -1,12 +1,17 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.*
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonType
 
 val ObjClassType by lazy { ObjClass("Class") }
 
 open class ObjClass(
     val className: String,
-    vararg val parents: ObjClass,
+    vararg parents: ObjClass,
 ) : Obj() {
 
+    var constructorMeta: ArgsDeclaration? = null
     var instanceConstructor: Statement? = null
 
     val allParentsSet: Set<ObjClass> =
@@ -18,6 +23,7 @@ open class ObjClass(
 
     // members: fields most often
     private val members = mutableMapOf<String, ObjRecord>()
+    private val classMembers = mutableMapOf<String, ObjRecord>()
 
     override fun toString(): String = className
 
@@ -32,10 +38,6 @@ open class ObjClass(
         return instance
     }
 
-    fun defaultInstance(): Obj = object : Obj() {
-        override val objClass: ObjClass = this@ObjClass
-    }
-
     fun createField(
         name: String,
         initialValue: Obj,
@@ -49,11 +51,28 @@ open class ObjClass(
         members[name] = ObjRecord(initialValue, isMutable, visibility)
     }
 
+    fun createClassField(
+        name: String,
+        initialValue: Obj,
+        isMutable: Boolean = false,
+        visibility: Visibility = Visibility.Public,
+        pos: Pos = Pos.builtIn
+    ) {
+        val existing = classMembers[name]
+        if( existing != null)
+            throw ScriptError(pos, "$name is already defined in $objClass or one of its supertypes")
+        classMembers[name] = ObjRecord(initialValue, isMutable, visibility)
+    }
+
     fun addFn(name: String, isOpen: Boolean = false, code: suspend Scope.() -> Obj) {
         createField(name, statement { code() }, isOpen)
     }
 
     fun addConst(name: String, value: Obj) = createField(name, value, isMutable = false)
+    fun addClassConst(name: String, value: Obj) = createClassField(name, value)
+    fun addClassFn(name: String, isOpen: Boolean = false, code: suspend Scope.() -> Obj) {
+        createClassField(name, statement { code() }, isOpen)
+    }
 
 
     /**
@@ -68,6 +87,19 @@ open class ObjClass(
     fun getInstanceMember(atPos: Pos, name: String): ObjRecord =
         getInstanceMemberOrNull(name)
             ?: throw ScriptError(atPos, "symbol doesn't exist: $name")
+
+    override suspend fun readField(scope: Scope, name: String): ObjRecord {
+        classMembers[name]?.let {
+            return it
+        }
+        return super.readField(scope, name)
+    }
+
+    override suspend fun invokeInstanceMethod(scope: Scope, name: String, args: Arguments): Obj {
+        return classMembers[name]?.value?.invoke(scope, this, args) ?: super.invokeInstanceMethod(scope, name, args)
+    }
+
+    open suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj = scope.raiseNotImplemented()
 }
 
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjCollection.kt

@@ -1,4 +1,4 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 /**
  * Collection is an iterator with `size`]

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjDuration.kt

@@ -0,0 +1,154 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.days
+import kotlin.time.Duration.Companion.hours
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.DurationUnit
+
+class ObjDuration(val duration: Duration) : Obj() {
+    override val objClass: ObjClass = type
+
+    override fun toString(): String {
+        return duration.toString()
+    }
+
+    override suspend fun compareTo(scope: Scope, other: Obj): Int {
+        return if( other is ObjDuration)
+            duration.compareTo(other.duration)
+        else -1
+    }
+
+    override fun hashCode(): Int {
+        return duration.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjDuration
+
+        return duration == other.duration
+    }
+
+    companion object {
+        val type = object : ObjClass("Duration") {
+            override suspend fun callOn(scope: Scope): Obj {
+                val args = scope.args
+                if( args.list.size > 1 )
+                    scope.raiseIllegalArgument("can't construct Duration(${args.inspect()})")
+                val a0 = args.list.getOrNull(0)
+
+                return ObjDuration(
+                    when (a0) {
+                        null -> Duration.ZERO
+                        is ObjInt -> a0.value.seconds
+                        is ObjReal -> a0.value.seconds
+                        else -> {
+                            scope.raiseIllegalArgument("can't construct Instant(${args.inspect()})")
+                        }
+                    }
+                )
+            }
+        }.apply {
+            addFn("days") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.DAYS).toObj()
+            }
+            addFn("hours") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.HOURS).toObj()
+            }
+            addFn("minutes") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.MINUTES).toObj()
+            }
+            addFn("seconds") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.SECONDS).toObj()
+            }
+            addFn("milliseconds") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.MILLISECONDS).toObj()
+            }
+            addFn("microseconds") {
+                thisAs<ObjDuration>().duration.toDouble(DurationUnit.MICROSECONDS).toObj()
+            }
+            // extensions
+
+            ObjInt.type.addFn("seconds") {
+                ObjDuration(thisAs<ObjInt>().value.seconds)
+            }
+
+            ObjInt.type.addFn("second") {
+                ObjDuration(thisAs<ObjInt>().value.seconds)
+            }
+            ObjInt.type.addFn("milliseconds") {
+                ObjDuration(thisAs<ObjInt>().value.milliseconds)
+            }
+
+            ObjInt.type.addFn("millisecond") {
+                ObjDuration(thisAs<ObjInt>().value.milliseconds)
+            }
+            ObjReal.type.addFn("seconds") {
+                ObjDuration(thisAs<ObjReal>().value.seconds)
+            }
+
+            ObjReal.type.addFn("second") {
+                ObjDuration(thisAs<ObjReal>().value.seconds)
+            }
+
+            ObjReal.type.addFn("milliseconds") {
+                ObjDuration(thisAs<ObjReal>().value.milliseconds)
+            }
+            ObjReal.type.addFn("millisecond") {
+                ObjDuration(thisAs<ObjReal>().value.milliseconds)
+            }
+
+            ObjInt.type.addFn("minutes") {
+                ObjDuration(thisAs<ObjInt>().value.minutes)
+            }
+            ObjReal.type.addFn("minutes") {
+                ObjDuration(thisAs<ObjReal>().value.minutes)
+            }
+            ObjInt.type.addFn("minute") {
+                ObjDuration(thisAs<ObjInt>().value.minutes)
+            }
+            ObjReal.type.addFn("minute") {
+                ObjDuration(thisAs<ObjReal>().value.minutes)
+            }
+            ObjInt.type.addFn("hours") {
+                ObjDuration(thisAs<ObjInt>().value.hours)
+            }
+            ObjReal.type.addFn("hours") {
+                ObjDuration(thisAs<ObjReal>().value.hours)
+            }
+            ObjInt.type.addFn("hour") {
+                ObjDuration(thisAs<ObjInt>().value.hours)
+            }
+            ObjReal.type.addFn("hour") {
+                ObjDuration(thisAs<ObjReal>().value.hours)
+            }
+            ObjInt.type.addFn("days") {
+                ObjDuration(thisAs<ObjInt>().value.days)
+            }
+            ObjReal.type.addFn("days") {
+                ObjDuration(thisAs<ObjReal>().value.days)
+            }
+            ObjInt.type.addFn("day") {
+                ObjDuration(thisAs<ObjInt>().value.days)
+            }
+            ObjReal.type.addFn("day") {
+                ObjDuration(thisAs<ObjReal>().value.days)
+            }
+
+
+//            addFn("epochSeconds") {
+//                val instant = thisAs<ObjInstant>().instant
+//                ObjReal(instant.epochSeconds + instant.nanosecondsOfSecond * 1e-9)
+//            }
+//            addFn("epochMilliseconds") {
+//                ObjInt(instant.toEpochMilliseconds())
+//            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstance.kt

@@ -1,4 +1,9 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Arguments
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 
 class ObjInstance(override val objClass: ObjClass) : Obj() {
 
@@ -41,6 +46,18 @@ class ObjInstance(override val objClass: ObjClass) : Obj() {
         return "${objClass.className}($fields)"
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        val meta = objClass.constructorMeta
+            ?: scope.raiseError("can't serialize non-serializable object (no constructor meta)")
+        for( p in meta.params) {
+            val r = readField(scope, p.name)
+            println("serialize ${p.name}=${r.value}")
+            TODO()
+//            encoder.encodeObj(scope, r.value)
+        }
+        // todo: possible vars?
+    }
+
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
         if (other !is ObjInstance) return -1
         if (other.objClass != objClass) return -1

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstant.kt

@@ -0,0 +1,178 @@
+package net.sergeych.lyng.obj
+
+import kotlinx.datetime.Clock
+import kotlinx.datetime.Instant
+import kotlinx.datetime.isDistantFuture
+import kotlinx.datetime.isDistantPast
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonSettings
+import net.sergeych.lynon.LynonType
+
+class ObjInstant(val instant: Instant,val truncateMode: LynonSettings.InstantTruncateMode=LynonSettings.InstantTruncateMode.Microsecond) : Obj() {
+    override val objClass: ObjClass get() = type
+
+    override fun toString(): String {
+        return instant.toString()
+    }
+
+    override suspend fun plus(scope: Scope, other: Obj): Obj {
+        return when (other) {
+            is ObjDuration -> ObjInstant(instant + other.duration)
+            else -> super.plus(scope, other)
+        }
+    }
+
+    override suspend fun minus(scope: Scope, other: Obj): Obj {
+        return when (other) {
+            is ObjDuration -> ObjInstant(instant - other.duration)
+            is ObjInstant -> ObjDuration(instant - other.instant)
+            else -> super.plus(scope, other)
+        }
+    }
+
+    override suspend fun compareTo(scope: Scope, other: Obj): Int {
+        if( other is ObjInstant) {
+            return instant.compareTo(other.instant)
+        }
+        return super.compareTo(scope, other)
+    }
+
+    override suspend fun toKotlin(scope: Scope): Any {
+        return instant
+    }
+
+    override fun hashCode(): Int {
+        return instant.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjInstant
+
+        return instant == other.instant
+    }
+
+    override suspend fun lynonType(): LynonType = LynonType.Instant
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.putBits(truncateMode.ordinal, 2)
+        when(truncateMode) {
+            LynonSettings.InstantTruncateMode.Millisecond ->
+                encoder.encodeSigned(instant.toEpochMilliseconds())
+            LynonSettings.InstantTruncateMode.Second ->
+                encoder.encodeSigned(instant.epochSeconds)
+            LynonSettings.InstantTruncateMode.Microsecond -> {
+                encoder.encodeSigned(instant.epochSeconds)
+                encoder.encodeUnsigned(instant.nanosecondsOfSecond.toULong() / 1000UL)
+            }
+        }
+    }
+
+    companion object {
+        val distantFuture by lazy {
+            ObjInstant(Instant.DISTANT_FUTURE)
+        }
+
+        val distantPast by lazy {
+            ObjInstant(Instant.DISTANT_PAST)
+        }
+
+        val type = object : ObjClass("Instant") {
+            override suspend fun callOn(scope: Scope): Obj {
+                val args = scope.args
+                val a0 = args.list.getOrNull(0)
+                return ObjInstant(
+                    when (a0) {
+                        null -> {
+                            val t = Clock.System.now()
+                            Instant.fromEpochSeconds(t.epochSeconds, t.nanosecondsOfSecond)
+                        }
+                        is ObjInt -> Instant.fromEpochSeconds(a0.value)
+                        is ObjReal -> {
+                            val seconds = a0.value.toLong()
+                            val nanos = (a0.value - seconds) * 1e9
+                            Instant.fromEpochSeconds(seconds, nanos.toLong())
+                        }
+                        is ObjInstant -> a0.instant
+
+                        else -> {
+                            scope.raiseIllegalArgument("can't construct Instant(${args.inspect()})")
+                        }
+                    }
+                )
+            }
+
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj {
+                val mode = LynonSettings.InstantTruncateMode.entries[decoder.getBitsAsInt(2)]
+                return when (mode) {
+                    LynonSettings.InstantTruncateMode.Microsecond -> ObjInstant(
+                        Instant.fromEpochSeconds(
+                            decoder.unpackSigned(), decoder.unpackUnsignedInt() * 1000
+                        )
+                    )
+                    LynonSettings.InstantTruncateMode.Millisecond -> ObjInstant(
+                        Instant.fromEpochMilliseconds(
+                            decoder.unpackSigned()
+                        )
+                    )
+                    LynonSettings.InstantTruncateMode.Second -> ObjInstant(
+                        Instant.fromEpochSeconds(decoder.unpackSigned())
+                    )
+                }
+            }
+
+        }.apply {
+            addFn("epochSeconds") {
+                val instant = thisAs<ObjInstant>().instant
+                ObjReal(instant.epochSeconds + instant.nanosecondsOfSecond * 1e-9)
+            }
+            addFn("isDistantFuture") {
+                thisAs<ObjInstant>().instant.isDistantFuture.toObj()
+            }
+            addFn("isDistantPast") {
+                thisAs<ObjInstant>().instant.isDistantPast.toObj()
+            }
+            addFn("epochWholeSeconds") {
+                ObjInt(thisAs<ObjInstant>().instant.epochSeconds)
+            }
+            addFn("nanosecondsOfSecond") {
+                ObjInt(thisAs<ObjInstant>().instant.nanosecondsOfSecond.toLong())
+            }
+            addFn("truncateToSecond") {
+                val t = thisAs<ObjInstant>().instant
+                ObjInstant(Instant.fromEpochSeconds(t.epochSeconds), LynonSettings.InstantTruncateMode.Second)
+            }
+            addFn("truncateToMillisecond") {
+                val t = thisAs<ObjInstant>().instant
+                ObjInstant(
+                    Instant.fromEpochSeconds(t.epochSeconds, t.nanosecondsOfSecond / 1_000_000 * 1_000_000),
+                    LynonSettings.InstantTruncateMode.Millisecond
+                )
+            }
+            addFn("truncateToMicrosecond") {
+                val t = thisAs<ObjInstant>().instant
+                ObjInstant(
+                    Instant.fromEpochSeconds(t.epochSeconds, t.nanosecondsOfSecond / 1_000 * 1_000),
+                    LynonSettings.InstantTruncateMode.Microsecond
+                )
+            }
+            // class members
+
+            addClassConst("distantFuture", distantFuture)
+            addClassConst("distantPast", distantPast)
+            addClassFn("now") {
+                ObjInstant(Clock.System.now())
+            }
+//            addFn("epochMilliseconds") {
+//                ObjInt(instant.toEpochMilliseconds())
+//            }
+        }
+
+    }
+}
+
+

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt

@@ -1,6 +1,11 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
-data class ObjInt(var value: Long) : Obj(), Numeric {
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
+
+class ObjInt(var value: Long, override val isConst: Boolean = false) : Obj(), Numeric {
     override val asStr get() = ObjString(value.toString())
     override val longValue get() = value
     override val doubleValue get() = value.toDouble()
@@ -14,18 +19,22 @@ data class ObjInt(var value: Long) : Obj(), Numeric {
     }
 
     override suspend fun getAndIncrement(scope: Scope): Obj {
+        ensureNotConst(scope)
         return ObjInt(value).also { value++ }
     }
 
     override suspend fun getAndDecrement(scope: Scope): Obj {
+        ensureNotConst(scope)
         return ObjInt(value).also { value-- }
     }
 
     override suspend fun incrementAndGet(scope: Scope): Obj {
+        ensureNotConst(scope)
         return ObjInt(++value)
     }
 
     override suspend fun decrementAndGet(scope: Scope): Obj {
+        ensureNotConst(scope)
         return ObjInt(--value)
     }
 
@@ -70,7 +79,7 @@ data class ObjInt(var value: Long) : Obj(), Numeric {
      * assignment
      */
     override suspend fun assign(scope: Scope, other: Obj): Obj? {
-        return if (other is ObjInt) {
+        return if (!isConst && other is ObjInt) {
             value = other.value
             this
         } else null
@@ -89,10 +98,42 @@ data class ObjInt(var value: Long) : Obj(), Numeric {
         return value == other.value
     }
 
+    override suspend fun negate(scope: Scope): Obj {
+        return ObjInt(-value)
+    }
+
+    override suspend fun lynonType(): LynonType = when (value) {
+        0L -> LynonType.Int0
+        else -> {
+            if (value > 0) LynonType.IntPositive
+            else LynonType.IntNegative
+        }
+    }
+
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        when (lynonType) {
+            null -> encoder.encodeSigned(value)
+            LynonType.Int0 -> {}
+            LynonType.IntPositive -> encoder.encodeUnsigned(value.toULong())
+            LynonType.IntNegative -> encoder.encodeUnsigned((-value).toULong())
+            else -> scope.raiseIllegalArgument("Unsupported lynon type code for Int: $lynonType")
+        }
+    }
+
     companion object {
-        val Zero = ObjInt(0)
-        val One = ObjInt(1)
-        val type = ObjClass("Int")
+        val Zero = ObjInt(0, true)
+        val One = ObjInt(1, true)
+        val type = object : ObjClass("Int") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
+                when (lynonType) {
+                    null -> ObjInt(decoder.unpackSigned())
+                    LynonType.Int0 -> Zero
+                    LynonType.IntPositive -> ObjInt(decoder.unpackUnsigned().toLong())
+                    LynonType.IntNegative -> ObjInt(-decoder.unpackUnsigned().toLong())
+                    else -> scope.raiseIllegalState("illegal type code for Int: $lynonType")
+                }
+        }
     }
 }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjIterable.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Arguments
+import net.sergeych.lyng.Statement
 
 /**
  * Abstract class that must provide `iterator` method that returns [ObjIterator] instance.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjIterator.kt

@@ -0,0 +1,3 @@
+package net.sergeych.lyng.obj
+
+val ObjIterator by lazy { ObjClass("Iterator") }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjKotlinIterator.kt

@@ -1,9 +1,10 @@
 @file:Suppress("unused")
 
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
+import net.sergeych.lyng.Scope
 
 /**
  * Iterator wrapper to allow Kotlin collections to be returned from Lyng objects;

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt

@@ -1,12 +1,10 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
 
 class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
 
-    init {
-        for (p in objClass.parents)
-            parentInstances.add(p.defaultInstance())
-    }
-
     override fun toString(): String = "[${
         list.joinToString(separator = ", ") { it.inspect() }
     }]"
@@ -51,9 +49,8 @@ class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
         }
     }
 
-    override suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
-        val i = index
-        list[i] = newValue
+    override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
+        list[index.toInt()] = newValue
     }
 
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
@@ -136,16 +133,6 @@ class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
                     (thisObj as ObjList).list.size.toObj()
                 }
             )
-            addFn("getAt") {
-                requireExactCount(1)
-                thisAs<ObjList>().getAt(this, requiredArg<Obj>(0))
-            }
-            addFn("putAt") {
-                requireExactCount(2)
-                val newValue = args[1]
-                thisAs<ObjList>().putAt(this, requiredArg<ObjInt>(0).value.toInt(), newValue)
-                newValue
-            }
             createField("add",
                 statement {
                     val l = thisAs<ObjList>().list

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjMap.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Statement
 
 class ObjMapEntry(val key: Obj, val value: Obj) : Obj() {
 
@@ -39,7 +42,11 @@ class ObjMap(val map: MutableMap<Obj, Obj> = mutableMapOf()) : Obj() {
     override val objClass = type
 
     override suspend fun getAt(scope: Scope, index: Obj): Obj =
-        map.getOrElse(index) { scope.raiseNoSuchElement() }
+        map.get(index) ?: ObjNull
+
+    override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
+        map[index] = newValue
+    }
 
     override suspend fun contains(scope: Scope, other: Obj): Boolean {
         return other in map
@@ -51,6 +58,19 @@ class ObjMap(val map: MutableMap<Obj, Obj> = mutableMapOf()) : Obj() {
     }
     override fun toString(): String = map.toString()
 
+    override fun hashCode(): Int {
+        return map.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjMap
+
+        return map == other.map
+    }
+
     companion object {
 
         suspend fun listToMap(scope: Scope, list: List<Obj>): MutableMap<Obj, Obj> {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjMutableBuffer.kt

@@ -0,0 +1,71 @@
+package net.sergeych.lyng.obj
+
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.toList
+import net.sergeych.lyng.Scope
+
+class ObjMutableBuffer(byteArray: UByteArray) : ObjBuffer(byteArray) {
+
+    override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
+        byteArray[checkIndex(scope, index.toObj())] = when (newValue) {
+            is ObjInt -> newValue.value.toUByte()
+            is ObjChar -> newValue.value.code.toUByte()
+            else -> scope.raiseIllegalArgument(
+                "invalid byte value for buffer at index ${index.inspect()}: ${newValue.inspect()}"
+            )
+        }
+    }
+
+    companion object {
+
+        private suspend fun createBufferFrom(scope: Scope, obj: Obj): ObjBuffer =
+            when (obj) {
+                is ObjBuffer -> ObjMutableBuffer(obj.byteArray.copyOf())
+                is ObjInt -> {
+                    if (obj.value < 0)
+                        scope.raiseIllegalArgument("buffer size must be positive")
+                    val data = UByteArray(obj.value.toInt())
+                    ObjMutableBuffer(data)
+                }
+
+                is ObjString -> ObjMutableBuffer(obj.value.encodeToByteArray().asUByteArray())
+                else -> {
+                    if (obj.isInstanceOf(ObjIterable)) {
+                        ObjMutableBuffer(
+                            obj.toFlow(scope).map { it.toLong().toUByte() }.toList().toTypedArray()
+                                .toUByteArray()
+                        )
+                    } else
+                        scope.raiseIllegalArgument(
+                            "can't construct buffer from ${obj.inspect()}"
+                        )
+                }
+            }
+
+        val type = object : ObjClass("MutableBuffer", ObjBuffer.type) {
+            override suspend fun callOn(scope: Scope): Obj {
+                val args = scope.args.list
+                return when (args.size) {
+                    // empty buffer
+                    0 -> ObjMutableBuffer(ubyteArrayOf())
+                    1 -> createBufferFrom(scope, args[0])
+                    else -> {
+                        // create buffer from array, each argument should be a byte then:
+                        val data = UByteArray(args.size)
+                        for ((i, b) in args.withIndex()) {
+                            val code = when (b) {
+                                is ObjChar -> b.value.code.toUByte()
+                                is ObjInt -> b.value.toUByte()
+                                else -> scope.raiseIllegalArgument(
+                                    "invalid byte value for buffer constructor at index $i: ${b.inspect()}"
+                                )
+                            }
+                            data[i] = code
+                        }
+                        ObjMutableBuffer(data)
+                    }
+                }
+            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRange.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Obj() {
 
@@ -15,6 +17,31 @@ class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Ob
         return result.toString()
     }
 
+    /**
+     * IF end is open (null/ObjNull), returns null
+     * Otherwise, return correct value for the exclusive end
+     * raises [ObjIllegalArgumentException] if end is not ObjInt
+     */
+    fun exclusiveIntEnd(scope: Scope): Int? =
+        if (end == null || end is ObjNull) null
+        else {
+            if (end !is ObjInt) scope.raiseIllegalArgument("end is not int")
+            if (isEndInclusive) end.value.toInt() + 1 else end.value.toInt()
+        }
+
+
+    /**
+     * If start is null/ObjNull, returns 0
+     * if start is not ObjInt, raises [ObjIllegalArgumentException]
+     * otherwise returns start.value.toInt()
+     */
+    fun startInt(scope: Scope): Int =
+        if( start == null || start is ObjNull) 0
+        else {
+            if( start is ObjInt) start.value.toInt()
+            else scope.raiseIllegalArgument("start is not Int: ${start.inspect()}")
+        }
+
     suspend fun containsRange(scope: Scope, other: ObjRange): Boolean {
         if (start != null) {
             // our start is not -∞ so other start should be GTE or is not contained:
@@ -74,6 +101,27 @@ class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Ob
             ?: -1
     }
 
+    override fun hashCode(): Int {
+        var result = start?.hashCode() ?: 0
+        result = 31 * result + (end?.hashCode() ?: 0)
+        result = 31 * result + isEndInclusive.hashCode()
+        return result
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjRange
+
+        if (start != other.start) return false
+        if (end != other.end) return false
+        if (isEndInclusive != other.isEndInclusive) return false
+
+        return true
+    }
+
+
     companion object {
         val type = ObjClass("Range", ObjIterable).apply {
             addFn("start") {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRangeIterator.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjRangeIterator(val self: ObjRange) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjReal.kt

@@ -1,5 +1,11 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import kotlin.math.floor
 import kotlin.math.roundToLong
 
@@ -56,8 +62,21 @@ data class ObjReal(val value: Double) : Obj(), Numeric {
         return value == other.value
     }
 
+    override suspend fun negate(scope: Scope): Obj {
+        return ObjReal(-value)
+    }
+
+    override suspend fun lynonType(): LynonType = LynonType.Real
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.encodeReal(value)
+    }
+
     companion object {
-        val type: ObjClass = ObjClass("Real").apply {
+        val type: ObjClass = object : ObjClass("Real") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
+                ObjReal(decoder.unpackDouble())
+        }.apply {
             createField(
                 "roundToInt",
                 statement(Pos.builtIn) {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRecord.kt

@@ -0,0 +1,18 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Visibility
+
+/**
+ * Record to store object with access rules, e.g. [isMutable] and access level [visibility].
+ */
+data class ObjRecord(
+    var value: Obj,
+    val isMutable: Boolean,
+    val visibility: Visibility = Visibility.Public,
+    var importedFrom: Scope? = null
+) {
+    @Suppress("unused")
+    fun qualifiedName(name: String): String =
+        "${importedFrom?.packageName ?: "anonymous"}.$name"
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjSet.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjSet(val set: MutableSet<Obj> = mutableSetOf()) : Obj() {
 
@@ -65,6 +67,19 @@ class ObjSet(val set: MutableSet<Obj> = mutableSetOf()) : Obj() {
         }
     }
 
+    override fun hashCode(): Int {
+        return set.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjSet
+
+        return set == other.set
+    }
+
     companion object {
 
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt

@@ -1,7 +1,12 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import net.sergeych.sprintf.sprintf
 
 @Serializable
@@ -15,7 +20,6 @@ data class ObjString(val value: String) : Obj() {
 //        return i
 //    }
 
-
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
         if (other !is ObjString) return -2
         return this.value.compareTo(other.value)
@@ -37,12 +41,12 @@ data class ObjString(val value: String) : Obj() {
     }
 
     override suspend fun getAt(scope: Scope, index: Obj): Obj {
-        if( index is ObjInt ) return ObjChar(value[index.toInt()])
-        if( index is ObjRange ) {
-            val start = if(index.start == null || index.start.isNull) 0 else  index.start.toInt()
-            val end = if( index.end  == null || index.end.isNull ) value.length else  {
+        if (index is ObjInt) return ObjChar(value[index.toInt()])
+        if (index is ObjRange) {
+            val start = if (index.start == null || index.start.isNull) 0 else index.start.toInt()
+            val end = if (index.end == null || index.end.isNull) value.length else {
                 val e = index.end.toInt()
-                if( index.isEndInclusive) e + 1 else e
+                if (index.isEndInclusive) e + 1 else e
             }
             return ObjString(value.substring(start, end))
         }
@@ -54,7 +58,10 @@ data class ObjString(val value: String) : Obj() {
     }
 
     override suspend fun callOn(scope: Scope): Obj {
-        return ObjString(this.value.sprintf(*scope.args.toKotlinList(scope).toTypedArray()))
+        return ObjString(this.value.sprintf(*scope.args
+            .toKotlinList(scope)
+            .map { if (it == null) "null" else it }
+            .toTypedArray()))
     }
 
     override suspend fun contains(scope: Scope, other: Obj): Boolean {
@@ -74,8 +81,21 @@ data class ObjString(val value: String) : Obj() {
         return value == other.value
     }
 
+    override suspend fun lynonType(): LynonType = LynonType.String
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        val data = value.encodeToByteArray()
+        encoder.encodeCached(data) { encoder.encodeBinaryData(data) }
+    }
+
+
     companion object {
-        val type = ObjClass("String").apply {
+        val type = object : ObjClass("String") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
+                    decoder.decodeCached {
+                        ObjString(decoder.unpackBinaryData().decodeToString())
+                    }
+        }.apply {
             addFn("toInt") {
                 ObjInt(thisAs<ObjString>().value.toLong())
             }
@@ -114,8 +134,14 @@ data class ObjString(val value: String) : Obj() {
             addFn("upper") {
                 thisAs<ObjString>().value.uppercase().let(::ObjString)
             }
+            addFn("characters") {
+                ObjList(
+                    thisAs<ObjString>().value.map { ObjChar(it) }.toMutableList()
+                )
+            }
+            addFn("encodeUtf8") { ObjBuffer(thisAs<ObjString>().value.encodeToByteArray().asUByteArray()) }
             addFn("size") { ObjInt(thisAs<ObjString>().value.length.toLong()) }
-            addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble())}
+            addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble()) }
         }
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportManager.kt

@@ -0,0 +1,141 @@
+package net.sergeych.lyng.pacman
+
+import net.sergeych.lyng.*
+import net.sergeych.synctools.ProtectedOp
+import net.sergeych.synctools.withLock
+
+/**
+ * Import manager allow to register packages with builder lambdas and act as an
+ * [ImportProvider]. Note that packages _must be registered_ first with [addPackage],
+ * [addSourcePackages] or [addTextPackages]. Registration is cheap, actual package
+ * building is lazily performed on [createModuleScope], when the package will
+ * be first imported.
+ *
+ * It is possible to register new packages at any time, but it is not allowed to override
+ * packages already registered.
+ */
+class ImportManager(
+    rootScope: Scope = Script.defaultImportManager.newModule(),
+    securityManager: SecurityManager = SecurityManager.allowAll
+) : ImportProvider(rootScope, securityManager) {
+
+    val packageNames: List<String> get() = imports.keys.toList()
+
+    private inner class Entry(
+        val packageName: String,
+        val builder: suspend (ModuleScope) -> Unit,
+        var cachedScope: ModuleScope? = null
+    ) {
+
+        suspend fun getScope(pos: Pos): ModuleScope {
+            cachedScope?.let { return it }
+            return ModuleScope(inner, pos, packageName).apply {
+                cachedScope = this
+                builder(this)
+            }
+        }
+    }
+
+
+    /**
+     * Inner provider does not lock [op], the only difference; it is meant to be used
+     * exclusively by the coroutine that starts actual import chain
+     */
+    private inner class InternalProvider : ImportProvider(rootScope) {
+        override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope {
+            return doImport(packageName, pos)
+        }
+
+        override fun getActualProvider(): ImportProvider {
+            return this@ImportManager
+        }
+    }
+
+    /**
+     * Inner module import provider used to prepare lazily prepared modules
+     */
+    private val inner = InternalProvider()
+
+
+    private val imports = mutableMapOf<String, Entry>()
+
+    val op = ProtectedOp()
+
+
+    /**
+     * Register new package that can be imported. It is not possible to unregister or
+     * update package already registered.
+     *
+     * Packages are lazily created when first imported somewhere, so the registration is
+     * cheap; the recommended procedure is to register all available packages prior to
+     * compile with this.
+     *
+     * @param name package name
+     * @param builder lambda to create actual package using the given [ModuleScope]
+     */
+    fun addPackage(name: String, builder: suspend (ModuleScope) -> Unit) {
+        op.withLock {
+            if (name in imports)
+                throw IllegalArgumentException("Package $name already exists")
+            imports[name] = Entry(name, builder)
+        }
+    }
+
+    /**
+     * Bulk [addPackage] with slightly better performance
+     */
+    @Suppress("unused")
+    fun addPackages(registrationData: List<Pair<String, suspend (ModuleScope) -> Unit>>) {
+        op.withLock {
+            for (pp in registrationData) {
+                if (pp.first in imports)
+                    throw IllegalArgumentException("Package ${pp.first} already exists")
+                imports[pp.first] = Entry(pp.first, pp.second)
+            }
+        }
+    }
+
+    /**
+     * Perform actual import or return ready scope. __It must only be called when
+     * [op] is locked__, e.g. only internally
+     */
+    private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
+        val entry = imports[packageName] ?: throw ImportException(pos, "package not found: $packageName")
+        return entry.getScope(pos)
+    }
+
+    override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope =
+        doImport(packageName, pos)
+
+    /**
+     * Add packages that only need to compile [Source].
+     */
+    fun addSourcePackages(vararg sources: Source) {
+        for (s in sources) {
+            addPackage(s.extractPackageName()) {
+                it.eval(s)
+            }
+
+        }
+    }
+
+    /**
+     * Add source packages using package name as [Source.fileName], for simplicity
+     */
+    fun addTextPackages(vararg sourceTexts: String) {
+        for (s in sourceTexts) {
+            var source = Source("tmp", s)
+            val packageName = source.extractPackageName()
+            source = Source(packageName, s)
+            addPackage(packageName) { it.eval(source) }
+        }
+    }
+
+    fun copy(): ImportManager =
+        op.withLock {
+            ImportManager(rootScope, securityManager).apply {
+                imports.putAll(this@ImportManager.imports)
+            }
+        }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/ImportProvider.kt

@@ -0,0 +1,51 @@
+package net.sergeych.lyng.pacman
+
+import net.sergeych.lyng.*
+
+/**
+ * Package manager INTERFACE (abstract class). Performs import routines
+ * using abstract [createModuleScope] method ot be implemented by heirs.
+ *
+ * Notice that [createModuleScope] is responsible for caching the modules;
+ * base class relies on caching. This is not implemented here as the correct
+ * caching strategy depends on the import provider
+ */
+abstract class ImportProvider(
+    val rootScope: Scope,
+    val securityManager: SecurityManager = SecurityManager.allowAll
+) {
+
+    open fun getActualProvider() = this
+
+    /**
+     * Find an import and create a scope for it. This method must implement caching so repeated
+     * imports are not repeatedly loaded and parsed and should be cheap.
+     *
+     * @throws ImportException if the module is not found
+     */
+    abstract suspend fun createModuleScope(pos: Pos,packageName: String): ModuleScope
+
+    /**
+     * Check that the import is possible and allowed. This method is called on compile time by [Compiler];
+     * actual module loading is performed by [ModuleScope.importInto]
+     */
+    suspend fun prepareImport(pos: Pos, name: String, symbols: Map<String, String>?): ModuleScope {
+        if (!securityManager.canImportModule(name))
+            throw ImportException(pos, "Module $name is not allowed")
+        symbols?.keys?.forEach {
+            if (!securityManager.canImportSymbol(name, it)) throw ImportException(
+                pos,
+                "Symbol $name.$it is not allowed"
+            )
+        }
+        return createModuleScope(pos, name)
+    }
+
+    fun newModule() = newModuleAt(Pos.builtIn)
+
+    fun newModuleAt(pos: Pos): ModuleScope =
+        ModuleScope(this, pos, "unknown")
+}
+
+
+

lynglib/src/commonMain/kotlin/net/sergeych/lyng/pacman/InlineSourcesImportProvider.kt

@@ -0,0 +1,35 @@
+package net.sergeych.lyng.pacman
+
+import kotlinx.coroutines.CompletableDeferred
+import net.sergeych.lyng.*
+import net.sergeych.mp_tools.globalLaunch
+
+/**
+ * The sample import provider that imports sources available in memory.
+ * on construction time.
+ *
+ * Actually it is left here only as a demo.
+ */
+class InlineSourcesImportProvider(sources: List<Source>,
+                                  rootScope: ModuleScope = Script.defaultImportManager.newModule(),
+                                  securityManager: SecurityManager = SecurityManager.allowAll
+) : ImportProvider(rootScope) {
+
+    private val manager = ImportManager(rootScope, securityManager)
+
+    private val readyManager = CompletableDeferred<ImportManager>()
+
+    /**
+     * This implementation only
+     */
+    override suspend fun createModuleScope(pos: Pos, packageName: String): ModuleScope {
+        return readyManager.await().createModuleScope(pos, packageName)
+    }
+
+    init {
+        globalLaunch {
+            manager.addSourcePackages(*sources.toTypedArray())
+            readyManager.complete(manager)
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/statements.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjClass
+
 fun String.toSource(name: String = "eval"): Source = Source(name, this)
 
 sealed class ObjType {
@@ -11,7 +14,7 @@ sealed class ObjType {
 @Suppress("unused")
 abstract class Statement(
     val isStaticConst: Boolean = false,
-    val isConst: Boolean = false,
+    override val isConst: Boolean = false,
     val returnType: ObjType = ObjType.Any
 ) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitInput.kt

@@ -0,0 +1,90 @@
+package net.sergeych.lynon
+
+interface BitInput {
+
+
+    fun getBitOrNull(): Int?
+
+    fun getBitsOrNull(count: Int): ULong? {
+        var result = 0UL
+        var resultMask = 1UL
+        for( i in 0 ..< count) {
+            when(getBitOrNull()) {
+                null -> return null
+                1 -> result = result or resultMask
+                0 -> {}
+            }
+            resultMask = resultMask shl 1
+        }
+        return result
+    }
+
+    fun getBits(count: Int): ULong {
+        return getBitsOrNull(count) ?: throw IllegalStateException("Unexpected end of stream")
+    }
+
+    fun getBit(): Int {
+        return getBitOrNull() ?: throw IllegalStateException("Unexpected end of stream")
+    }
+
+    fun unpackUnsigned(): ULong =
+        unpackUnsignedOrNull() ?: throw IllegalStateException("Unexpected end of stream")
+
+    fun unpackUnsignedOrNull(): ULong? {
+        val tetrades = getBitsOrNull(4)?.toInt() ?: return null
+        var result = 0UL
+        var shift = 0
+        for (i in 0.. tetrades) {
+            result = result or (getBits(4) shl shift)
+            shift += 4
+        }
+        return result
+    }
+
+    fun unpackSigned(): Long {
+        val isNegative = getBit()
+        val value = unpackUnsigned().toLong()
+        return if( isNegative == 1) -value else value
+    }
+
+    @Suppress("unused")
+    fun getBool(): Boolean {
+        return getBit() == 1
+    }
+
+    fun getBytes(count: Int): ByteArray? {
+        val result = ByteArray(count)
+        for (i in 0..<count) {
+            val b = getBitsOrNull(8) ?: return null
+            result[i] = b.toByte()
+        }
+        return result
+    }
+
+
+    fun decompress(): ByteArray = decompressOrNull() ?: throw DecompressionException("Unexpected end of stream")
+
+    fun decompressOrNull(): ByteArray? {
+        val originalSize = unpackUnsignedOrNull()?.toInt() ?: return null
+        return if( getBit() == 1) {
+            // data is compressed
+//            val expectedCRC = getBits(32).toUInt()
+            val method = getBits(2).toInt()
+            if( method != 0) throw DecompressionException("Unknown compression method")
+            LZW.decompress(this, originalSize).asByteArray()
+        }
+        else {
+            getBytes(originalSize) ?: throw DecompressionException("Unexpected end of stream in uncompressed data")
+        }
+    }
+
+    @Suppress("unused")
+    fun decompressStringOrNull(): String? = decompressOrNull()?.decodeToString()
+
+    fun decompressString(): String = decompress().decodeToString()
+    fun unpackDouble(): Double {
+        val bits = getBits(64)
+        return Double.fromBits(bits.toLong())
+    }
+}
+

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitList.kt

@@ -0,0 +1,34 @@
+package net.sergeych.lynon
+
+@Suppress("unused")
+interface BitList {
+    operator fun get(bitIndex: Long): Int
+    operator fun set(bitIndex: Long,value: Int)
+    val size: Long
+    val indices: LongRange
+
+    fun toInput(): BitInput = object : BitInput {
+        private var index = 0L
+
+        override fun getBitOrNull(): Int? =
+            if( index < size) this@BitList[index++]
+            else null
+    }
+}
+
+fun bitListOf(vararg bits: Int): BitList {
+    return if( bits.size > 64) {
+        BitArray.ofBits(*bits)
+    }
+    else
+        TinyBits.of(*bits)
+}
+
+@Suppress("unused")
+fun bitListOfSize(sizeInBits: Long): BitList {
+    return if( sizeInBits > 64) {
+        BitArray.withBitSize(sizeInBits)
+    }
+    else
+        TinyBits()
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitOutput.kt

@@ -0,0 +1,104 @@
+package net.sergeych.lynon
+
+interface BitOutput {
+
+    fun putBits(bits: ULong, count: Int) {
+        require(count <= 64)
+        var x = bits
+        for (i in 0..<count) {
+            putBit((x and 1u).toInt())
+            x = x shr 1
+        }
+    }
+
+    fun putBits(bits: Int, count: Int) {
+        require(count <= 32)
+        var x = bits
+        for (i in 0..<count) {
+            putBit((x and 1))
+            x = x shr 1
+        }
+    }
+
+    fun putBit(bit: Int)
+
+    fun putBits(bitList: BitList) {
+        for (i in bitList.indices)
+            putBit(bitList[i])
+    }
+
+    fun packUnsigned(value: ULong) {
+        val tetrades = sizeInTetrades(value)
+        putBits(tetrades - 1, 4)
+        var rest = value
+        for (i in 0..<tetrades) {
+            putBits(rest and 0xFu, 4)
+            rest = rest shr 4
+        }
+    }
+
+    @Suppress("unused")
+    fun packSigned(value: Long) {
+        if (value < 0) {
+            putBit(1)
+            packUnsigned((-value).toULong())
+        } else {
+            putBit(0)
+            packUnsigned(value.toULong())
+        }
+    }
+
+    fun putBytes(data: ByteArray) {
+        for (b in data) {
+            putBits(b.toULong(), 8)
+        }
+    }
+
+
+    /**
+     * Create compressed record with content and size check. Compression works with _bytes_.
+     *
+     * Structure:
+     *
+     * | size | meaning                                          |
+     * |------|--------------------------------------------------|
+     * | packed unsigned | size of uncompressed content in bytes |
+     * | 1               | 0 - not compressed, 1 - compressed |
+     *
+     * __If compressed__, then:
+     *
+     * | size | meaning                              |
+     * |------|--------------------------------------|
+     * |  2   | 00 - LZW, other combinations reserved|
+     *
+     * After this header compressed bits follow.
+     *
+     * __If not compressed,__ then source data follows as bit stream.
+     *
+     * Compressed block overhead is 3 bits, uncompressed 1.
+     */
+    fun compress(source: ByteArray) {
+        // size
+        packUnsigned(source.size.toULong())
+        // check compression is effective?
+        val compressed = LZW.compress(source.asUByteArray())
+        // check that compression is effective including header bits size:
+        if( compressed.size + 2 < source.size * 8L) {
+            println("write compressed")
+            putBit(1)
+            // LZW algorithm
+            putBits(0, 2)
+            // compressed data
+            putBits(compressed)
+        }
+        else {
+            putBit(0)
+            putBytes(source)
+        }
+    }
+
+    fun compress(source: String) {
+        compress(source.encodeToByteArray())
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/DecompressionException.kt

@@ -0,0 +1,3 @@
+package net.sergeych.lynon
+
+class DecompressionException(message: String) : IllegalArgumentException(message) {}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -0,0 +1,67 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjClass
+
+open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSettings.default) {
+
+    fun getBitsAsInt(bitsSize: Int): Int {
+        return bin.getBits(bitsSize).toInt()
+    }
+
+    fun unpackUnsignedInt(): Int = bin.unpackUnsigned().toInt()
+
+    fun decompress() = bin.decompress()
+
+    val cache = mutableListOf<Any>()
+
+
+    inline fun <T : Any>decodeCached(f: LynonDecoder.() -> T): T {
+        return if (bin.getBit() == 0) {
+            // unpack and cache
+            f().also {
+                if (settings.shouldCache(it)) cache.add(it)
+            }
+        } else {
+            // get cache reference
+            val size = sizeInBits(cache.size)
+            val id = bin.getBitsOrNull(size)?.toInt()
+                ?: throw RuntimeException("Invalid object id: unexpected end of stream")
+            if (id >= cache.size) throw RuntimeException("Invalid object id: $id should be in 0..<${cache.size}")
+            @Suppress("UNCHECKED_CAST")
+            cache[id] as T
+        }
+    }
+
+    suspend fun decodeAny(scope: Scope): Obj = decodeCached {
+        val type = LynonType.entries[bin.getBits(4).toInt()]
+        type.objClass.deserialize(scope, this, type)
+    }
+
+    suspend fun decodeObject(scope: Scope, type: ObjClass): Obj {
+        return decodeCached {  type.deserialize(scope, this, null) }
+    }
+
+    fun unpackBinaryData(): ByteArray = bin.decompress()
+
+    @Suppress("unused")
+    fun unpackBinaryDataOrNull(): ByteArray? = bin.decompressOrNull()
+
+    fun unpackBoolean(): Boolean {
+        return bin.getBit() == 1
+    }
+
+    fun unpackDouble(): Double {
+        return Double.fromBits(bin.getBits(64).toLong())
+    }
+
+    fun unpackSigned(): Long {
+        return bin.unpackSigned()
+    }
+
+    fun unpackUnsigned(): ULong {
+        return bin.unpackUnsigned()
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -0,0 +1,108 @@
+package net.sergeych.lynon
+
+import net.sergeych.bintools.ByteChunk
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.*
+
+enum class LynonType(val objClass: ObjClass,val defaultFrequency: Int = 1) {
+    Null(ObjNull.objClass, 80),
+    Int0(ObjInt.type, 70),
+    IntNegative(ObjInt.type, 50),
+    IntPositive(ObjInt.type, 100),
+    String(ObjString.type, 100),
+    Real(ObjReal.type),
+    Bool(ObjBool.type, 80),
+    List(ObjList.type, 70),
+    Map(ObjMap.type,40),
+    Set(ObjSet.type),
+    Buffer(ObjBuffer.type, 50),
+    Instant(ObjInstant.type, 30),
+    Duration(ObjDuration.type),
+    Other(Obj.rootObjectType,60);
+}
+
+open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = LynonSettings.default) {
+
+    val cache = mutableMapOf<Any, Int>()
+
+    suspend fun encodeCached(item: Any, packer: suspend LynonEncoder.() -> Unit) {
+
+        suspend fun serializeAndCache(key: Any = item) {
+            cache[key]?.let { cacheId ->
+                val size = sizeInBits(cache.size)
+                bout.putBit(1)
+                bout.putBits(cacheId.toULong(), size)
+            } ?: run {
+                bout.putBit(0)
+                if (settings.shouldCache(item))
+                    cache[key] = cache.size
+                packer()
+            }
+        }
+
+        when (item) {
+            is ByteArray -> serializeAndCache(ByteChunk(item.asUByteArray()))
+            is UByteArray -> serializeAndCache(ByteChunk(item))
+            else -> serializeAndCache(item)
+        }
+    }
+
+    /**
+     * Encode any Lyng object [Obj], which can be serialized, using type record. This allow to
+     * encode any object with the overhead of type record.
+     *
+     * Caching is used automatically.
+     */
+    suspend fun encodeAny(scope: Scope, value: Obj) {
+        encodeCached(value) {
+            val type = value.lynonType()
+            putType(type)
+            value.serialize(scope, this, type)
+        }
+    }
+
+    private fun putType(type: LynonType) {
+        bout.putBits(type.ordinal.toULong(), 4)
+    }
+
+    suspend fun encodeObject(scope: Scope, obj: Obj) {
+        encodeCached(obj) {
+            obj.serialize(scope, this, null)
+        }
+    }
+
+    fun encodeBinaryData(data: ByteArray) {
+        bout.compress(data)
+    }
+
+    fun encodeSigned(value: Long) {
+        bout.packSigned(value)
+    }
+
+    @Suppress("unused")
+    fun encodeUnsigned(value: ULong) {
+        bout.packUnsigned(value)
+    }
+
+    @Suppress("unused")
+    fun encodeBool(value: Boolean) {
+        bout.putBit(if (value) 1 else 0)
+    }
+
+    fun encodeReal(value: Double) {
+        bout.putBits(value.toRawBits().toULong(), 64)
+    }
+
+    fun encodeBoolean(value: Boolean) {
+        bout.putBit(if (value) 1 else 0)
+    }
+
+    fun putBits(value: Int, sizeInBits: Int) {
+        bout.putBits(value.toULong(), sizeInBits)
+    }
+
+    fun putBit(bit: Int) {
+        bout.putBit(bit)
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonSettings.kt

@@ -0,0 +1,29 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.obj.ObjBool
+import net.sergeych.lyng.obj.ObjChar
+import net.sergeych.lyng.obj.ObjInt
+import net.sergeych.lyng.obj.ObjNull
+import kotlin.math.absoluteValue
+
+open class LynonSettings {
+    enum class InstantTruncateMode {
+        Second,
+        Millisecond,
+        Microsecond
+    }
+
+    open fun shouldCache(obj: Any): Boolean = when (obj) {
+        is ObjChar -> false
+        is ObjInt -> obj.value.absoluteValue > 0x10000FF
+        is ObjBool -> false
+        is ObjNull -> false
+        is ByteArray -> obj.size > 2
+        is UByteArray -> obj.size > 2
+        else -> true
+    }
+
+    companion object {
+        val default = LynonSettings()
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/MemoryBitInput.kt

@@ -0,0 +1,40 @@
+package net.sergeych.lynon
+
+
+class MemoryBitInput(val packedBits: UByteArray, val lastByteBits: Int) : BitInput {
+
+    constructor(ba: BitArray) : this(ba.bytes, ba.lastByteBits) {}
+    constructor(mba: MemoryBitOutput) : this(mba.toBitArray()) {}
+
+    private var index = 0
+
+    private var isEndOfStream: Boolean = packedBits.isEmpty() || (packedBits.size == 1 && lastByteBits == 0)
+        private set
+
+    /**
+     * Return next byte, int in 0..255 range, or -1 if end of stream reached
+     */
+    private var accumulator = if( isEndOfStream ) 0 else packedBits[0].toInt()
+
+    private var bitCounter = 0
+
+    override fun getBitOrNull(): Int? {
+        if (isEndOfStream) return null
+        val result = accumulator and 1
+        accumulator = accumulator shr 1
+        bitCounter++
+        // is end?
+        if( index == packedBits.lastIndex && bitCounter == lastByteBits ) {
+            isEndOfStream = true
+        }
+        else {
+            if( bitCounter == 8 ) {
+                bitCounter = 0
+                accumulator = packedBits[++index].toInt()
+            }
+        }
+        return result
+    }
+
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/MemoryBitOutput.kt

@@ -0,0 +1,157 @@
+package net.sergeych.lynon
+
+import kotlin.math.min
+
+/**
+ * BitList implementation as fixed suze array of bits; indexing works exactly same as if
+ * [MemoryBitInput] is used with [MemoryBitInput.getBit]. See [MemoryBitOutput] for
+ * bits order and more information.
+ */
+class BitArray(val bytes: UByteArray, val lastByteBits: Int) : BitList {
+
+    val bytesSize: Int get() = bytes.size
+
+    override val size by lazy { bytes.size * 8L - (8 - lastByteBits) }
+
+    override val indices by lazy { 0..<size }
+
+    /**
+     * @return [BitInput] that can be used to read from this array
+     */
+    fun toBitInput(): BitInput = MemoryBitInput(bytes, lastByteBits)
+
+    private fun getIndexAndMask(bitIndex: Long): Pair<Int, Int> {
+        val byteIndex = (bitIndex / 8).toInt()
+        if (byteIndex !in bytes.indices)
+            throw IndexOutOfBoundsException("$bitIndex is out of bounds")
+        val i = (bitIndex % 8).toInt()
+        if (byteIndex == bytes.lastIndex && i >= lastByteBits)
+                throw IndexOutOfBoundsException("$bitIndex is out of bounds (last)")
+        return byteIndex to (1 shl i)
+    }
+
+    override operator fun get(bitIndex: Long): Int =
+        getIndexAndMask(bitIndex).let { (byteIndex, mask) ->
+            if (bytes[byteIndex].toInt() and mask == 0) 0 else 1
+        }
+
+    override operator fun set(bitIndex: Long, value: Int) {
+        require(value == 0 || value == 1)
+        val (byteIndex, mask) = getIndexAndMask(bitIndex)
+        if (value == 1)
+            bytes[byteIndex] = bytes[byteIndex] or mask.toUByte()
+        else
+            bytes[byteIndex] = bytes[byteIndex] and mask.inv().toUByte()
+    }
+
+    override fun toString(): String {
+        val result = StringBuilder()
+        val s = min(size, 64)
+        for (i in 0..<s) result.append(this[i])
+        if (s < size) result.append("…")
+        return result.toString()
+    }
+
+    @Suppress("unused")
+    fun asByteArray(): ByteArray = bytes.asByteArray()
+
+    @Suppress("unused")
+    fun asUbyteArray(): UByteArray = bytes
+
+    companion object {
+
+        fun withBitSize(size: Long): BitArray {
+            val byteSize = ((size + 7) / 8).toInt()
+            val lastByteBits = size % 8
+            return BitArray(UByteArray(byteSize), lastByteBits.toInt())
+        }
+
+        fun ofBits(vararg bits: Int): BitArray {
+            return withBitSize(bits.size.toLong()).apply {
+                for (i in bits.indices) {
+                    this[i.toLong()] = bits[i]
+                }
+            }
+        }
+    }
+
+}
+
+/**
+ * [BitOutput] implementation that writes to a memory buffer, LSB first.
+ *
+ * Bits are stored in the least significant bits of the bytes. E.g. the first bit
+ * added by [putBit] will be stored in the bit 0x01 of the first byte, the second bit
+ * in the bit 0x02 of the first byte, etc.
+ *
+ * This allows automatic fill of the last byte with zeros. This is important when
+ * using bytes stored from [asByteArray] or [asUbyteArray]. When converting to
+ * bytes, automatic padding to byte size is applied. With such bit order, constructing
+ * [BitInput] to read from [ByteArray.toUByteArray] result only provides 0 to 7 extra zeroes bits
+ * at teh end which is often acceptable. To avoid this, use [toBitArray]; the [BitArray]
+ * stores exact number of bits and [BitArray.toBitInput] provides [BitInput] that
+ * decodes exactly same bits.
+ *
+ */
+class MemoryBitOutput : BitOutput {
+    private val buffer = mutableListOf<UByte>()
+
+    private var accumulator = 0
+
+    private var mask = 1
+
+    override fun putBit(bit: Int) {
+        when (bit) {
+            0 -> {}
+            1 -> accumulator = accumulator or mask
+            else -> throw IllegalArgumentException("Bit must be 0 or 1")
+        }
+        mask = mask shl 1
+        if(mask == 0x100) {
+            mask = 1
+            outputByte(accumulator.toUByte())
+            accumulator = accumulator shr 8
+        }
+    }
+
+    var isClosed = false
+        private set
+
+    fun close(): BitArray {
+        if (!isClosed) {
+            if (mask != 0x01) {
+                outputByte(accumulator.toUByte())
+            }
+            isClosed = true
+        }
+        return toBitArray()
+    }
+
+    fun lastBits(): Int {
+        check(isClosed)
+        return when(mask) {
+            0x01 -> 8   // means that all bits of the last byte are in use
+            0x02 -> 1
+            0x04 -> 2
+            0x08 -> 3
+            0x10 -> 4
+            0x20 -> 5
+            0x40 -> 6
+            0x80 -> 7
+            else -> throw IllegalStateException("Invalid state, mask=${mask.toString(16)}")
+        }
+    }
+
+    fun toBitArray(): BitArray {
+        if (!isClosed) {
+            close()
+        }
+        return BitArray(buffer.toTypedArray().toUByteArray(), lastBits())
+    }
+
+    fun toBitInput(): BitInput = toBitArray().toBitInput()
+
+    private fun outputByte(byte: UByte) {
+        buffer.add(byte)
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/TinyBits.kt

@@ -0,0 +1,71 @@
+package net.sergeych.lynon
+
+
+/**
+ * Bit size-aware code, short [BitList] implementation, up to 64 bits (efficiency tradeoff).
+ * E.g `Bits(0, 3) != Bits(0, 2). For longer, use [BitArray].
+ *
+ * Note that [bitListOf] creates [TinyBits] when possible.
+ */
+class TinyBits(initValue: ULong = 0U, override val size: Long = 0): BitList {
+
+    private var bits: ULong = initValue
+
+    constructor(value: ULong, size: Int): this(value, size.toLong()) {}
+
+    override val indices: LongRange by lazy { 0..<size }
+
+    override operator fun get(bitIndex: Long): Int {
+        if( bitIndex !in indices) throw IndexOutOfBoundsException("index out of bounds: $bitIndex")
+        val mask = 1UL shl (size - bitIndex - 1).toInt()
+        return if (bits and mask != 0UL) 1 else 0
+    }
+
+    override fun set(bitIndex: Long, value: Int) {
+        val mask = 1UL shl (size - bitIndex - 1).toInt()
+        if( value == 1)
+            bits = bits or mask
+        else
+            bits = bits and mask.inv()
+    }
+
+    override fun toString(): String {
+        val result = StringBuilder()
+        for (i in 0..<size) result.append(this[i])
+        return result.toString()
+    }
+
+    val value by ::bits
+
+    /**
+     * Add bit shifting value to the left and return _new instance_
+     */
+    fun insertBit(bit: Int): TinyBits {
+        return TinyBits((bits shl 1) or bit.toULong(), size + 1)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as TinyBits
+
+        if (size != other.size) return false
+        if (bits != other.bits) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = size.hashCode()
+        result = 31 * result + bits.hashCode()
+        return result
+    }
+
+
+    companion object {
+        fun of(vararg bits: Int): TinyBits {
+            return TinyBits(0UL, bits.size).apply { bits.forEachIndexed { i, v -> this[i.toLong()] = v } }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/bit_tools.kt

@@ -0,0 +1,35 @@
+package net.sergeych.lynon
+
+/**
+ * Hoq many tetrades needed to store the value. It is faster to use this function
+ * than to use sizeInBits
+ *
+ * Size for 0 is 1
+ */
+fun sizeInTetrades(value: ULong): Int {
+    if( value == 0UL ) return 1
+    var size = 0
+    var rest = value
+    while( rest != 0UL ) {
+        size++
+        rest = rest shr 4
+    }
+    return size
+}
+
+/**
+ * How many bits needed to store the value. Size for 0 is 1,
+ */
+@Suppress("unused")
+fun sizeInBits(value: ULong): Int {
+    if( value == 0UL ) return 1
+    var size = 0
+    var rest = value
+    while( rest != 0UL ) {
+        size++
+        rest = rest shr 1
+    }
+    return size
+}
+
+fun sizeInBits(value: Int): Int = sizeInBits(value.toULong())

lynglib/src/commonMain/kotlin/net/sergeych/lynon/huffman.kt

@@ -0,0 +1,274 @@
+package net.sergeych.lynon
+
+import net.sergeych.collections.SortedList
+import net.sergeych.lynon.Huffman.Alphabet
+
+
+/**
+ * Generic huffman encoding implementation using bits input/output and abstract [Alphabet].
+ */
+object Huffman {
+
+    /**
+     * Alphabet interface: source can be variable bit size codes, not just bytes,
+     * so the Huffman encoding is not limited to bytes. It works with any alphabet
+     * using its _ordinals_; encoding between source symbols and ordinals are
+     * performed by the alphabet. See [byteAlphabet] for example.
+     */
+    interface Alphabet<T> {
+        val maxOrdinal: Int
+
+        /**
+         * Write correct symbol for the [ordinal] to the [bout]. This is
+         * the inverse of [ordinalOf] but as [T] could be variable bit size,
+         * we provide output bit stream.
+         */
+        fun decodeOrdinalTo(bout: BitOutput, ordinal: Int)
+
+        /**
+         * Find the ordinal of the source symbol
+         */
+        fun ordinalOf(value: T): Int
+
+        operator fun get(ordinal: Int): T
+    }
+
+    /**
+     * Alphabet for unsigned bytes, allows to encode bytes easily
+     */
+    val byteAlphabet = object : Alphabet<UByte> {
+        override val maxOrdinal: Int
+            get() = 256
+
+        override fun decodeOrdinalTo(bout: BitOutput, ordinal: Int) {
+            bout.putBits(ordinal, 8)
+        }
+
+        override fun ordinalOf(value: UByte): Int = value.toInt()
+
+        override operator fun get(ordinal: Int): UByte = ordinal.toUByte()
+    }
+
+    sealed class Node(val freq: Int) : Comparable<Node> {
+        override fun compareTo(other: Node): Int {
+            return freq.compareTo(other.freq)
+        }
+
+        abstract fun decodeOrdinal(bin: BitInput): Int?
+
+        class Leaf(val ordinal: Int, freq: Int) : Node(freq) {
+            override fun toString(): String {
+                return "[$ordinal:$freq]"
+            }
+
+            override fun decodeOrdinal(bin: BitInput): Int {
+                return ordinal//.also { println(": ${Char(value)}") }
+            }
+        }
+
+        class Internal(val left: Node, val right: Node) : Node(left.freq + right.freq) {
+            override fun toString(): String {
+                return "[${left.freq}<- :<$freq>: ->${right.freq}]"
+            }
+
+            override fun decodeOrdinal(bin: BitInput): Int? {
+                return when (bin.getBitOrNull().also { print("$it") }) {
+                    1 -> left.decodeOrdinal(bin)
+                    0 -> right.decodeOrdinal(bin)
+                    else -> null
+                }
+            }
+        }
+    }
+
+    data class Code(val ordinal: Int, val bits: TinyBits) {
+
+        val size by bits::size
+
+        override fun toString(): String {
+            return "[$ordinal:$size:$bits]"
+        }
+
+    }
+
+    private fun generateCanonicCodes(tree: Node, alphabet: Alphabet<*>): List<Code?> {
+        val codes = MutableList<Code?>(alphabet.maxOrdinal) { null }
+
+        fun traverse(node: Node, code: TinyBits) {
+            when (node) {
+                is Node.Leaf ->
+                    codes[node.ordinal] = (Code(node.ordinal, code))
+
+                is Node.Internal -> {
+                    traverse(node.left, code.insertBit(1))
+                    traverse(node.right, code.insertBit(0))
+                }
+            }
+        }
+        traverse(tree, TinyBits())
+
+        return makeCanonical(codes, alphabet)
+    }
+
+    private fun makeCanonical(source: List<Code?>,alphabet: Alphabet<*>): List<Code?> {
+        val sorted = source.filterNotNull().sortedWith(canonicComparator)
+
+        val canonical = MutableList<Code?>(alphabet.maxOrdinal) { null }
+
+        val first = sorted[0]
+        val prevValue = first.copy(bits = TinyBits(0UL, first.bits.size))
+        canonical[first.ordinal] = prevValue
+        var prev = prevValue.bits
+
+        for (i in 1..<sorted.size) {
+            var bits = TinyBits(prev.value + 1U, prev.size)
+            val code = sorted[i]
+            while (code.bits.size > bits.size) {
+                bits = bits.insertBit(0)
+            }
+            canonical[code.ordinal] = code.copy(bits = bits)//.also { println("$it") }
+            prev = bits
+        }
+        return canonical
+    }
+
+    private val canonicComparator = { a: Code, b: Code ->
+        if (a.bits.size == b.bits.size) {
+            a.ordinal.compareTo(b.ordinal)
+        } else {
+            a.bits.size.compareTo(b.bits.size)
+        }
+    }
+
+    private fun buildTree(data: Iterable<Int>,alphabet: Alphabet<*>): Node {
+        val frequencies = buildFrequencies(alphabet, data)
+        return buildTree(frequencies)
+    }
+
+    private fun buildTree(frequencies: Array<Int>): Node {
+//        println(data.toDump())
+
+        val list: SortedList<Node> = SortedList(*frequencies.mapIndexed { index, frequency -> Node.Leaf(index, frequency) }.filter { it.freq > 0 }
+            .toTypedArray())
+
+        // build the tree
+        while (list.size > 1) {
+            val left = list.removeAt(0)
+            val right = list.removeAt(0)
+            list.add(Node.Internal(left, right))
+        }
+        return list[0]
+    }
+
+    private fun buildFrequencies(
+        alphabet: Alphabet<*>,
+        data: Iterable<Int>
+    ): Array<Int> {
+        val maxOrdinal = alphabet.maxOrdinal
+        val frequencies = Array(maxOrdinal) { 0 }
+        data.forEach { frequencies[it]++ }
+        return frequencies
+    }
+
+    fun decompressUsingCodes(bin: BitInput, codes: List<Code?>, alphabet: Alphabet<*>): BitArray {
+        val result = MemoryBitOutput()
+        val table = codes.filterNotNull().associateBy { it.bits }
+
+        outer@ while (true) {
+            var input = TinyBits()
+            while (true) {
+                bin.getBitOrNull()?.let { input = input.insertBit(it) }
+                    ?: break@outer
+                val data = table[input]
+                if (data != null) {
+//                    println("Code found: ${data.bits} -> [${data.symbol.toChar()}]")
+                    alphabet.decodeOrdinalTo(result,data.ordinal)
+                    break
+                }
+            }
+        }
+        return result.toBitArray()
+    }
+
+    private fun serializeCanonicCodes(bout: BitOutput, codes: List<Code?>) {
+        var minSize: Int? = null
+        var maxSize: Int? = null
+        for (i in 1..<codes.size) {
+            val s = codes[i]?.size?.toInt() ?: continue
+            if (minSize == null || s < minSize) minSize = s
+            if (maxSize == null || s > maxSize) maxSize = s
+        }
+        val size = maxSize!! - minSize!! + 1
+        val sizeInBits = sizeInBits(size)
+        bout.packUnsigned(minSize.toULong())
+        bout.packUnsigned(sizeInBits.toULong())
+        for (c in codes) {
+            if (c != null)
+                bout.putBits(c.bits.size.toInt() - minSize + 1, sizeInBits)
+            else
+                bout.putBits(0, sizeInBits)
+        }
+    }
+
+    fun deserializeCanonicCodes(bin: BitInput, alphabet: Alphabet<*>): List<Code?> {
+        val minSize = bin.unpackUnsigned().toInt()
+        val sizeInBits = bin.unpackUnsigned().toInt()
+        val sorted = mutableListOf<Code>().also { codes ->
+            for (i in 0..<alphabet.maxOrdinal) {
+                val s = bin.getBits(sizeInBits).toInt()
+                if (s > 0) {
+                    codes.add(Code(i, TinyBits(0U, s - 1 + minSize)))
+                }
+            }
+        }.sortedWith(canonicComparator)
+
+        val result = MutableList<Code?>(alphabet.maxOrdinal) { null }
+        var prev = sorted[0].copy(bits = TinyBits(0U, sorted[0].bits.size))
+        result[prev.ordinal] = prev
+
+        for (i in 1..<sorted.size) {
+            val code = sorted[i]
+            var bits = TinyBits(prev.bits.value + 1u, prev.bits.size)
+            while (bits.size < code.bits.size) bits = bits.insertBit(0)
+            result[code.ordinal] = code.copy(bits = bits).also {
+                prev = it
+            }
+        }
+        return result
+    }
+
+//    fun generateCanonicalCodes(frequencies: Iterable<Int>): List<Code?> {
+//
+//    }
+
+    fun generateCanonicalCodes(frequencies: Array<Int>,alphabet: Alphabet<*>): List<Code?> =
+        generateCanonicCodes(buildTree(frequencies), alphabet)
+
+    fun <T>compress(plain: Iterable<T>,alphabet: Alphabet<T>): BitArray {
+
+        val source = plain.map { alphabet.ordinalOf(it) }
+        val root = buildTree(source,alphabet)
+
+        val codes = generateCanonicCodes(root, alphabet)
+
+        // serializa table
+
+        // test encode:
+        val bout = MemoryBitOutput()
+        serializeCanonicCodes(bout, codes)
+        for (i in source) {
+            val code = codes[i]!!
+//            println(">> $code")
+            bout.putBits(code.bits)
+        }
+//        println(bout.toBitArray().bytes.toDump())
+        val compressed = bout.toBitArray()
+        return compressed
+    }
+
+    fun <T>decompress(bin: BitInput,alphabet: Alphabet<T>): UByteArray {
+        val codes = deserializeCanonicCodes(bin, alphabet)
+        return decompressUsingCodes(bin, codes, alphabet).asUbyteArray()
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw.kt

@@ -0,0 +1,128 @@
+package net.sergeych.lynon
+
+import net.sergeych.bintools.ByteChunk
+import kotlin.math.roundToInt
+
+/**
+ * LZW lightweight pure kotlin compression.
+ */
+object LZW {
+
+    val MAX_CODE_SIZE = 17
+    val STOP_CODE = (1 shl MAX_CODE_SIZE) - 1
+    val MAX_DICT_SIZE = (STOP_CODE * 0.92).roundToInt()
+
+
+    fun compress(input: ByteArray, bitOutput: BitOutput)
+        = compress(input.asUByteArray(), bitOutput)
+
+    /**
+     * Compresses the input string using LZW algorithm
+     * @param input The string to compress
+     * @return List of compressed codes
+     */
+    fun compress(input: UByteArray, bitOutput: BitOutput) {
+        // Initialize dictionary with all possible single characters
+        val dictionary = mutableMapOf<ByteChunk, Int>()
+        for (i in 0..255) {
+            // 23
+            dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
+        }
+
+        var nextCode = 256
+        var current = ByteChunk(ubyteArrayOf())
+//            val result = mutableListOf<Int>()
+
+        for (char in input) {
+            val combined = current + char
+            if (dictionary.containsKey(combined)) {
+                current = combined
+            } else {
+                val size = sizeInBits(dictionary.size)
+                bitOutput.putBits(dictionary[current]!!, size)
+                if (dictionary.size >= MAX_DICT_SIZE) {
+                    bitOutput.putBits(STOP_CODE, size)
+                    dictionary.clear()
+                    nextCode = 256
+                    for (i in 0..255) {
+                        dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
+                    }
+                } else
+                    dictionary[combined] = nextCode++
+                current = ByteChunk(ubyteArrayOf(char))
+            }
+        }
+
+        if (current.size > 0) {
+            val size = sizeInBits(dictionary.size)
+            bitOutput.putBits(dictionary[current]!!, size)
+        }
+    }
+
+    fun compress(input: UByteArray): BitArray {
+        return MemoryBitOutput().apply {
+            compress(input, this)
+        }.toBitArray()
+    }
+
+    /**
+     * Decompresses a list of LZW codes back to the original string. Note that usage of apriori existing
+     * size is crucial: it let repeal explosion style attacks.
+     *
+     * @param compressed The list of compressed codes
+     * @param resultSize The expected size of the decompressed string
+     *
+     * @throws DecompressionException if something goes wrong
+     * @return The decompressed string
+     */
+    fun decompress(compressed: BitInput, resultSize: Int): UByteArray {
+        // Initialize dictionary with all possible single characters
+        val dictionary = mutableMapOf<Int, UByteArray>()
+        for (i in 0..255) {
+            dictionary[i] = ubyteArrayOf(i.toUByte())
+        }
+
+        var nextCode = 256
+        val firstCode = compressed.getBits(9).toInt()
+        var previous = dictionary[firstCode]
+            ?: throw DecompressionException("Invalid first compressed code: $firstCode")
+        val result = mutableListOf<UByte>()
+        result += previous
+
+        while (result.size < resultSize) {
+            val codeSize = sizeInBits(nextCode + 1)
+            val code = compressed.getBitsOrNull(codeSize)?.toInt() ?: break
+
+            if (code == STOP_CODE) {
+                nextCode = 256
+                dictionary.clear()
+                for (i in 0..255)
+                    dictionary[i] = ubyteArrayOf(i.toUByte())
+                previous = dictionary[compressed.getBits(9).toInt()]!!
+            } else {
+
+                val current = if (code in dictionary) {
+                    dictionary[code]!!
+                } else if (code == nextCode) {
+                    // Special case for pattern like cScSc
+                    previous + previous[0]
+                } else {
+                    throw DecompressionException("Invalid compressed code: $code")
+                }
+
+                result += current
+                dictionary[nextCode++] = previous + current[0]
+                previous = current
+            }
+        }
+
+        if (result.size != resultSize)
+            throw DecompressionException("Decompressed size is not equal to expected: real/expected = ${result.size}/$resultSize")
+        return result.toTypedArray().toUByteArray()
+    }
+}
+
+
+private operator fun ByteChunk.plus(byte: UByte): ByteChunk {
+    return ByteChunk(data + byte)
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/packer.kt

@@ -0,0 +1,36 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjBitBuffer
+import net.sergeych.lyng.obj.ObjClass
+import net.sergeych.lyng.obj.ObjString
+
+// Most often used types:
+
+
+val ObjLynonClass = object : ObjClass("Lynon") {
+
+    suspend fun Scope.encodeAny(obj: Obj): Obj {
+        val bout = MemoryBitOutput()
+        val serializer = LynonEncoder(bout)
+        serializer.encodeAny(this, obj)
+        return ObjBitBuffer(bout.toBitArray())
+    }
+
+    suspend fun Scope.decodeAny(source: Obj): Obj {
+        if( source !is ObjBitBuffer) throw Exception("Invalid source: $source")
+        val bin = source.bitArray.toInput()
+        val deserializer = LynonDecoder(bin)
+        return deserializer.decodeAny(this)
+    }
+
+}.apply {
+    addClassConst("test", ObjString("test_const"))
+    addClassFn("encode") {
+        encodeAny(requireOnlyArg<Obj>())
+    }
+    addClassFn("decode") {
+        decodeAny(requireOnlyArg<Obj>())
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt

@@ -0,0 +1,15 @@
+package net.sergeych.lynon
+
+/**
+ * Variant of [LynonEncoder] that writes to embedded [MemoryBitOutput]
+ */
+class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput(), settings: LynonSettings = LynonSettings.default)
+    : LynonEncoder(bout, settings) {
+}
+
+/**
+ * Variant of [LynonDecoder] that reads from a given `source` using [MemoryBitInput]
+ */
+class LynonUnpacker(source: BitInput) : LynonDecoder(source) {
+    constructor(packer: LynonPacker) : this((packer.bout as MemoryBitOutput).toBitInput())
+}

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -1,8 +1,10 @@
-
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.test.runTest
 import net.sergeych.lyng.*
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.InlineSourcesImportProvider
 import kotlin.test.*
 
 class ScriptTest {
@@ -604,7 +606,7 @@ class ScriptTest {
     }
 
     @Test
-    fun testAssignArgumentsmiddleEllipsis() = runTest {
+    fun testAssignArgumentsMiddleEllipsis() = runTest {
         val ttEnd = Token.Type.RBRACE
         val pa = ArgsDeclaration(
             listOf(
@@ -706,19 +708,13 @@ class ScriptTest {
             var t1 = 10
             outer@ while( t1 > 0 ) {
                 var t2 = 10
-                println("starting t2 = " + t2)
                 while( t2 > 0 ) {
                     t2 = t2 - 1
-                    println("t2 " + t2 + " t1 " + t1)
                     if( t2 == 3 && t1 == 7) {
-                        println("will break")
                         break@outer "ok2:"+t2+":"+t1
                     }
                 }
-                println("next t1")
-                t1 = t1 - 1
-                println("t1 now "+t1)
-                t1
+                --t1
             }
         """.trimIndent()
             ).toString()
@@ -733,8 +729,6 @@ class ScriptTest {
                 """
                 val count = 3
                 val res = if( count > 10 ) "too much" else "just " + count
-                println(count)
-                println(res)
                 res
                 """.trimIndent()
             )
@@ -770,7 +764,7 @@ class ScriptTest {
     fun testDecr() = runTest {
         val c = Scope()
         c.eval("var x = 9")
-        assertEquals(9, c.eval("x--").toInt())
+        assertEquals(9, c.eval("println(x); val a = x--; println(x); println(a); a").toInt())
         assertEquals(8, c.eval("x--").toInt())
         assertEquals(7, c.eval("x--").toInt())
         assertEquals(6, c.eval("x--").toInt())
@@ -2250,18 +2244,21 @@ class ScriptTest {
 
     @Test
     fun testLet() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x=0,y=0)
             assert( Point() is Object) 
             Point().let { println(it.x, it.y) }
             val x = null
             x?.let { println(it.x, it.y) }
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testApply() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x,y)
             // see the difference: apply changes this to newly created Point:
             val p = Point(1,2).apply { 
@@ -2270,12 +2267,14 @@ class ScriptTest {
             assertEquals(p, Point(2,3))
             >>> void
 
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testApplyThis() = runTest {
-        eval("""
+        eval(
+            """
             class Point(x,y)
             // see the difference: apply changes this to newly created Point:
             val p = Point(1,2).apply { 
@@ -2284,12 +2283,14 @@ class ScriptTest {
             assertEquals(p, Point(2,3))
             >>> void
 
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
     fun testExtend() = runTest() {
-        eval("""
+        eval(
+            """
             
             fun Int.isEven() {
                 this % 2 == 0
@@ -2311,7 +2312,8 @@ class ScriptTest {
             assert( 12.1.isInteger() == false )
             assert( "5".isInteger() )
             assert( ! "5.2".isInteger() )
-        """.trimIndent())
+        """.trimIndent()
+        )
     }
 
     @Test
@@ -2319,33 +2321,35 @@ class ScriptTest {
         val c = Scope()
         val arr = c.eval("[1,2,3]")
         // array is iterable so we can:
-        assertEquals(listOf(1,2,3),  arr.toFlow(c).map { it.toInt() }.toList())
+        assertEquals(listOf(1, 2, 3), arr.toFlow(c).map { it.toInt() }.toList())
     }
 
     @Test
     fun testAssociateBy() = runTest() {
-        eval("""
+        eval(
+            """
             val m = [123, 456].associateBy { "k:%s"(it) }
             println(m)
             assertEquals(123, m["k:123"])
             assertEquals(456, m["k:456"])
-            """)
-        listOf(1,2,3).associateBy { it * 10 }
+            """
+        )
+        listOf(1, 2, 3).associateBy { it * 10 }
     }
 
-    @Test
-    fun testImports1() = runTest() {
-        val foosrc = """
-            package lyng.foo
-            
-            fun foo() { "foo1" }
-            """.trimIndent()
-        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
-        assertNotNull(pm.modules.await()["lyng.foo"])
-        assertIs<ModuleScope>(pm.modules.await()["lyng.foo"]!!.await())
+//    @Test
+//    fun testImports1() = runTest() {
+//        val foosrc = """
+//            package lyng.foo
+//
+//            fun foo() { "foo1" }
+//            """.trimIndent()
+//        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
+//        assertNotNull(pm.modules["lyng.foo"])
+//        assertIs<ModuleScope>(pm.modules["lyng.foo"]!!.deferredModule.await())
 
-        assertEquals("foo1", pm.modules.await()["lyng.foo"]!!.await().eval("foo()").toString())
-    }
+//        assertEquals("foo1", pm.modules["lyng.foo"]!!.deferredModule.await().eval("foo()").toString())
+//    }
 
     @Test
     fun testImports2() = runTest() {
@@ -2354,7 +2358,7 @@ class ScriptTest {
             
             fun foo() { "foo1" }
             """.trimIndent()
-        val pm = InlineSourcesPacman(Pacman.emptyAllowAll, listOf(Source("foosrc", foosrc)))
+        val pm = InlineSourcesImportProvider(listOf(Source("foosrc", foosrc)))
 
         val src = """
             import lyng.foo
@@ -2365,4 +2369,292 @@ class ScriptTest {
         val scope = ModuleScope(pm, src)
         assertEquals("foo1", scope.eval(src).toString())
     }
+
+    @Test
+    fun testImports3() = runTest {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesImportProvider(
+            listOf(
+                Source("barsrc", barsrc),
+                Source("foosrc", foosrc),
+            )
+        )
+
+        val src = """
+            import lyng.foo
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
+    @Test
+    fun testImportsCircular() = runTest {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            import lyng.foo
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesImportProvider(
+            listOf(
+                Source("barsrc", barsrc),
+                Source("foosrc", foosrc),
+            )
+        )
+
+        val src = """
+            import lyng.bar
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
+    @Test
+    fun testDefaultImportManager() = runTest {
+        val scope = Scope.new()
+        assertFails {
+            scope.eval("""
+                import foo
+                foo()
+                """.trimIndent())
+        }
+        scope.importManager.addTextPackages("""
+            package foo
+            
+            fun foo() { "bar" }
+        """.trimIndent())
+        scope.eval("""
+                import foo
+                assertEquals( "bar", foo())
+                """.trimIndent())
+    }
+
+    @Test
+    fun testMaps() = runTest {
+        eval(
+            """
+                val map = Map( "a" => 1, "b" => 2 )
+                assertEquals( 1, map["a"] )
+                assertEquals( 2, map["b"] )
+                assertEquals( null, map["c"] )
+                map["c"] = 3
+                assertEquals( 3, map["c"] )
+        """.trimIndent()
+        )
+    }
+
+    @Test
+    fun testBuffer() = runTest {
+        eval("""
+            import lyng.buffer
+            
+            assertEquals( 0, Buffer().size )
+            assertEquals( 3, Buffer(1, 2, 3).size )
+            assertEquals( 5, Buffer("hello").size )
+            
+            var buffer = Buffer("Hello")
+            assertEquals( 5, buffer.size)
+            assertEquals('l'.code, buffer[2] )
+            assertEquals('l'.code, buffer[3] )
+            assertEquals("Hello", buffer.decodeUtf8())
+            
+            buffer = buffer.toMutable()
+            
+            buffer[2] = 101
+            assertEquals(101, buffer[2])
+            assertEquals("Heelo", buffer.decodeUtf8())
+            
+        """.trimIndent())
+    }
+
+    @Test
+    fun testBufferCompare() = runTest {
+        eval("""
+            import lyng.buffer
+            
+            println("Hello".characters())
+            val b1 = Buffer("Hello")
+            val b2 = Buffer("Hello".characters())
+            
+            assertEquals( b1, b2 )
+            val b3 = b1 + Buffer("!")
+            assertEquals( "Hello!", b3.decodeUtf8())
+            assert( b3 > b1 )
+            assert( b1 !== b2)
+            
+            val map = Map( b1 => "foo")
+            assertEquals("foo",  map[b1])
+            assertEquals("foo",  map[b2])
+            assertEquals(null,  map[b3])
+            
+        """.trimIndent())
+    }
+
+    @Test
+    fun testInstant() = runTest {
+        eval(
+            """
+            import lyng.time
+            
+            val now = Instant()
+//            assertEquals( now.epochSeconds, Instant(now.epochSeconds).epochSeconds )
+
+            assert( 10.seconds is Duration )
+            assertEquals( 10.seconds, Duration(10) )
+            assertEquals( 10.milliseconds, Duration(0.01) )
+            assertEquals( 10.milliseconds, 0.01.seconds )
+            assertEquals( 1001.5.milliseconds, 1.0015.seconds )
+
+            val n1 = now + 7.seconds
+            assert( n1 is Instant )
+
+            assertEquals( n1 - now, 7.seconds )
+            assertEquals( now - n1, -7.seconds )
+            
+            """.trimIndent()
+        )
+        delay(1000)
+    }
+    @Test
+    fun testTimeStatics() = runTest {
+        eval(
+            """
+            import lyng.time
+            assert( 100.minutes is Duration )
+            assert( 100.days is Duration )
+            assert( 1.day == 24.hours )
+            assert( 1.day.hours == 24 )
+            assert( 1.hour.seconds == 3600 )
+            assert( 1.minute.milliseconds == 60_000 )
+            
+            assert(Instant.distantFuture is Instant)
+            assert(Instant.distantPast is Instant)
+            assert( Instant.distantFuture - Instant.distantPast > 70_000_000.days)
+            val maxRange = Instant.distantFuture - Instant.distantPast
+            println("всего лет %g"(maxRange.days/365.2425))
+            """.trimIndent()
+        )
+    }
+
+    @Test
+    fun testInstantFormatting() = runTest {
+        eval(
+            """
+            import lyng.time
+            val now = Instant()
+            val unixEpoch = "%ts"(now)
+            println("current seconds is %s"(unixEpoch))
+            println("current time is %tT"(now))
+            assertEquals( unixEpoch.toInt(), now.epochSeconds.toInt() )
+            """.trimIndent()
+        )
+    }
+
+    @Test
+    fun testDoubleImports() = runTest {
+        val s = Scope.new()
+        println(Script.defaultImportManager.packageNames)
+        println(s.importManager.packageNames)
+
+        s.importManager.addTextPackages("""
+            package foo
+             
+            import lyng.time
+            
+            fun foo() {
+                println("foo: %s"(Instant()))
+            }
+        """.trimIndent())
+        s.importManager.addTextPackages("""
+            package bar
+             
+            import lyng.time
+            
+            fun bar() {
+                println("bar: %s"(Instant()))
+            }
+        """.trimIndent())
+
+        println(s.importManager.packageNames)
+
+        s.eval("""
+            import foo
+            import bar
+            
+            foo()
+            bar()
+            
+        """.trimIndent())
+
+    }
+
+    @Test
+    fun testIndexIntIncrements() = runTest {
+        eval("""
+        val x = [1,2,3]
+        x[1]++
+        ++x[0]
+        assertEquals( [2,3,3], x )
+        
+        import lyng.buffer
+        
+        val b = MutableBuffer(1,2,3)
+        b[1]++
+        assert( b == Buffer(1,3,3) )
+        ++b[0]
+        assertEquals( b, Buffer(2,3,3) )
+        """.trimIndent())
+    }
+
+    @Test
+    fun testIndexIntDecrements() = runTest {
+        eval("""
+        val x = [1,2,3]
+        x[1]--
+        --x[0]
+        assertEquals( [0,1,3], x )
+        
+        import lyng.buffer
+        
+        val b = Buffer(1,2,3).toMutable()
+        b[1]--
+        assert( b == Buffer(1,1,3) )
+        --b[0]
+        assertEquals( b, Buffer(0,1,3) )
+        """.trimIndent())
+    }
+
+    @Test
+    fun testRangeToList() = runTest {
+        val x = eval("""(1..10).toList()""") as ObjList
+        assertEquals(listOf(1,2,3,4,5,6,7,8,9,10), x.list.map { it.toInt() })
+        val y = eval("""(-2..3).toList()""") as ObjList
+        println(y.list)
+    }
+
 }

lynglib/src/jvmTest/kotlin/BookTest.kt

@@ -2,9 +2,10 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.flowOn
+import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.test.runTest
-import net.sergeych.lyng.ObjVoid
 import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.ObjVoid
 import java.nio.file.Files
 import java.nio.file.Files.readAllLines
 import java.nio.file.Paths
@@ -105,6 +106,10 @@ fun parseDocTests(fileName: String, bookMode: Boolean = false): Flow<DocTest> =
                         } else {
                             var isValid = true
                             val result = mutableListOf<String>()
+
+                            // remove empty trails:
+                            while( block.last().isEmpty() ) block.removeLast()
+
                             while (block.size > outStart) {
                                 val line = block.removeAt(outStart)
                                 if (!line.startsWith(">>> ")) {
@@ -265,6 +270,11 @@ class BookTest {
         runDocTests("../docs/Map.md")
     }
 
+    @Test
+    fun testBuffer() = runTest {
+        runDocTests("../docs/Buffer.md")
+    }
+
     @Test
     fun testSampleBooks() = runTest {
         for (bt in Files.list(Paths.get("../docs/samples")).toList()) {
@@ -283,4 +293,10 @@ class BookTest {
     fun testExceptionsBooks() = runTest {
         runDocTests("../docs/exceptions_handling.md")
     }
+
+    @Test
+    fun testTimeBooks() = runBlocking {
+        runDocTests("../docs/time.md")
+    }
+
 }

lynglib/src/jvmTest/kotlin/LynonTests.kt

@@ -0,0 +1,478 @@
+import junit.framework.TestCase.*
+import kotlinx.coroutines.test.runTest
+import net.sergeych.bintools.encodeToHex
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.eval
+import net.sergeych.lyng.obj.*
+import net.sergeych.lynon.*
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.test.Test
+import kotlin.test.assertContentEquals
+class LynonTests {
+
+    @Test
+    fun testSizeInTetrades() {
+        assertEquals(1, sizeInTetrades(0u))
+        assertEquals(1, sizeInTetrades(1u))
+        assertEquals(1, sizeInTetrades(15u))
+        assertEquals(2, sizeInTetrades(16u))
+        assertEquals(2, sizeInTetrades(254u))
+        assertEquals(2, sizeInTetrades(255u))
+        assertEquals(3, sizeInTetrades(256u))
+        assertEquals(3, sizeInTetrades(257u))
+    }
+
+    @Test
+    fun testSizeInBits() {
+        assertEquals(1, sizeInBits(0u))
+        assertEquals(1, sizeInBits(1u))
+        assertEquals(2, sizeInBits(2u))
+        assertEquals(2, sizeInBits(3u))
+        assertEquals(4, sizeInBits(15u))
+    }
+
+    @Test
+    fun testBitOutputSmall() {
+        val bout = MemoryBitOutput()
+        bout.putBit(1)
+        bout.putBit(1)
+        bout.putBit(0)
+        bout.putBit(1)
+        val x = bout.toBitArray()
+        assertEquals(1, x[0])
+        assertEquals(1, x[1])
+        assertEquals(0, x[2])
+        assertEquals(1, x[3])
+        assertEquals(4, x.size)
+        assertEquals("1101", x.toString())
+        val bin = MemoryBitInput(x)
+        assertEquals(1, bin.getBit())
+        assertEquals(1, bin.getBit())
+        assertEquals(0, bin.getBit())
+        assertEquals(1, bin.getBit())
+        assertEquals(null, bin.getBitOrNull())
+    }
+    @Test
+    fun testBitOutputMedium() {
+        val bout = MemoryBitOutput()
+        bout.putBit(1)
+        bout.putBit(1)
+        bout.putBit(0)
+        bout.putBit(1)
+        bout.putBits( 0, 7)
+        bout.putBits( 3, 2)
+        val x = bout.toBitArray()
+        assertEquals(1, x[0])
+        assertEquals(1, x[1])
+        assertEquals(0, x[2])
+        assertEquals(1, x[3])
+        assertEquals(13, x.size)
+        assertEquals("1101000000011", x.toString())
+        println(x.bytes.encodeToHex())
+        val bin = MemoryBitInput(x)
+        assertEquals(1, bin.getBit())
+        assertEquals(1, bin.getBit())
+        assertEquals(0, bin.getBit())
+        assertEquals(1, bin.getBit())
+
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+//        assertEquals(0, bin.getBit())
+        assertEquals(0UL, bin.getBits(7))
+        assertEquals(3UL, bin.getBits(2))
+        assertEquals(null, bin.getBitOrNull())
+    }
+
+    @Test
+    fun testBitStreams() {
+
+        val bout = MemoryBitOutput()
+        bout.putBits(2, 3)
+        bout.putBits(1, 7)
+        bout.putBits(197, 8)
+        bout.putBits(3, 4)
+        bout.close()
+
+        val bin = MemoryBitInput(bout)
+        assertEquals(2UL, bin.getBits(3))
+        assertEquals(1UL, bin.getBits(7))
+        assertEquals(197UL, bin.getBits(8))
+        assertEquals(3UL, bin.getBits(4))
+    }
+
+    @Test
+    fun testUnsignedPackInteger() {
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(1471792UL)
+        bout.close()
+        val bin = MemoryBitInput(bout)
+        assertEquals(1471792UL, bin.unpackUnsigned())
+    }
+
+    @Test
+    fun testUnsignedPackLongInteger() {
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(ULong.MAX_VALUE)
+        bout.close()
+        val bin = MemoryBitInput(bout)
+        assertEquals(ULong.MAX_VALUE, bin.unpackUnsigned())
+    }
+
+    @Test
+    fun testUnsignedPackLongSmallInteger() {
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(7UL)
+        bout.close()
+        val bin = MemoryBitInput(bout)
+        assertEquals(7UL, bin.unpackUnsigned())
+    }
+
+    @Test
+    fun testSignedPackInteger() {
+        val bout = MemoryBitOutput()
+        bout.packSigned(-1471792L)
+        bout.packSigned(1471792L)
+//        bout.packSigned(147179L)
+        bout.close()
+        val bin = MemoryBitInput(bout)
+        assertEquals(-1471792L, bin.unpackSigned())
+        assertEquals(1471792L, bin.unpackSigned())
+    }
+
+    @Test
+    fun testCache1() = runTest {
+        val bout = MemoryBitOutput()
+        val encoder = LynonEncoder(bout)
+        val s = "Hello, World!".toObj()
+        val scope = Scope()
+        encoder.encodeObject(scope, s) // 1
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s)
+        encoder.encodeObject(scope, s) // 8
+
+        val decoder = LynonDecoder(MemoryBitInput(bout))
+        val s1 = decoder.decodeObject(scope, ObjString.type) // 1
+        assertEquals(s, s1)
+        assertNotSame(s, s1)
+        val s2 = decoder.decodeObject(scope, ObjString.type)
+        assertEquals(s, s2)
+        assertSame(s1, s2)
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type))
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type))
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type))
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type))
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type))
+        assertSame(s1, decoder.decodeObject(scope, ObjString.type)) // 8
+    }
+
+    @Test
+    fun testCache2() = runTest {
+        val variants = (100..500).map { "Sample $it".toObj() }.shuffled()
+        var source = variants.shuffled()
+        for (i in 0..300) source += variants.shuffled()
+        val encoder = LynonPacker()
+        val scope = Scope()
+        for (s in source) {
+            encoder.encodeObject(scope, s)
+        }
+        val decoder = LynonUnpacker(encoder)
+        val restored = mutableListOf<Obj>()
+        for (i in source.indices) {
+            restored.add(decoder.decodeObject(scope, ObjString.type))
+        }
+        assertEquals(restored, source)
+    }
+
+    @Test
+    fun testUnpackBoolean() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            encodeObject(scope, ObjBool(true))
+            encodeObject(scope, ObjBool(false))
+            encodeObject(scope, ObjBool(true))
+            encodeObject(scope, ObjBool(true))
+        })
+        assertEquals(ObjTrue, decoder.decodeObject(scope, ObjBool.type))
+        assertEquals(ObjFalse, decoder.decodeObject(scope, ObjBool.type))
+        assertEquals(ObjTrue, decoder.decodeObject(scope, ObjBool.type))
+        assertEquals(ObjTrue, decoder.decodeObject(scope, ObjBool.type))
+    }
+
+    @Test
+    fun testUnpackReal() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            encodeObject(scope, ObjReal(-Math.PI))
+            encodeObject(scope, ObjReal(Math.PI))
+            encodeObject(scope, ObjReal(-Math.PI))
+            encodeObject(scope, ObjReal(Math.PI))
+            encodeObject(scope, ObjReal(Double.NaN))
+            encodeObject(scope, ObjReal(Double.NEGATIVE_INFINITY))
+            encodeObject(scope, ObjReal(Double.POSITIVE_INFINITY))
+            encodeObject(scope, ObjReal(Double.MIN_VALUE))
+            encodeObject(scope, ObjReal(Double.MAX_VALUE))
+        })
+        assertEquals(ObjReal(-Math.PI), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Math.PI), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(-Math.PI), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Math.PI), decoder.decodeObject(scope, ObjReal.type))
+        assert((decoder.decodeObject(scope, ObjReal.type)).toDouble().isNaN())
+        assertEquals(ObjReal(Double.NEGATIVE_INFINITY), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.POSITIVE_INFINITY), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.MIN_VALUE), decoder.decodeObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.MAX_VALUE), decoder.decodeObject(scope, ObjReal.type))
+    }
+    @Test
+    fun testUnpackInt() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            encodeObject(scope, ObjInt(0))
+            encodeObject(scope, ObjInt(-1))
+            encodeObject(scope, ObjInt(23))
+            encodeObject(scope, ObjInt(Long.MIN_VALUE))
+            encodeObject(scope, ObjInt(Long.MAX_VALUE))
+            encodeObject(scope, ObjInt(Long.MAX_VALUE))
+        })
+        assertEquals(ObjInt(0), decoder.decodeObject(scope, ObjInt.type))
+        assertEquals(ObjInt(-1), decoder.decodeObject(scope, ObjInt.type))
+        assertEquals(ObjInt(23), decoder.decodeObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MIN_VALUE), decoder.decodeObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MAX_VALUE), decoder.decodeObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MAX_VALUE), decoder.decodeObject(scope, ObjInt.type))
+    }
+
+    @Test
+    fun testLastvalue() {
+        var bin = MemoryBitInput(MemoryBitOutput().apply {
+            putBits(5, 3)
+        })
+        assertEquals(5UL, bin.getBits(3))
+        assertEquals(null, bin.getBitsOrNull(3))
+        bin = MemoryBitInput(MemoryBitOutput().apply {
+            putBits(5, 3)
+            putBits(1024, 11)
+            putBits(2, 2)
+        })
+        assertEquals(5UL, bin.getBits(3))
+        assertEquals(1024UL, bin.getBits(11))
+        assertEquals(2UL, bin.getBits(2))
+        assertEquals(null, bin.getBitsOrNull(3))
+    }
+
+
+    val original = Files.readString(Path.of("../sample_texts/dikkens_hard_times.txt"))
+
+    @Test
+    fun testEncodeNullsAndInts() = runTest{
+        testScope().eval("""
+            testEncode(null)
+            testEncode(0)
+        """.trimIndent())
+    }
+
+    @Test
+    fun testBufferEncoderInterop() = runTest{
+        val bout = MemoryBitOutput()
+        bout.putBits(0, 1)
+        bout.putBits(1, 4)
+        val bin = MemoryBitInput(bout.toBitArray().bytes, 8)
+        assertEquals(0UL, bin.getBits(1))
+        assertEquals(1UL, bin.getBits(4))
+    }
+
+    suspend fun testScope() =
+        Scope().apply { eval("""
+            import lyng.serialization
+            fun testEncode(value) {
+                val encoded = Lynon.encode(value)
+                println(encoded.toDump())
+                println("Encoded size %d: %s"(encoded.size, value))
+                assertEquals( value, Lynon.decode(encoded) )
+            }
+            """.trimIndent())
+        }
+
+
+    @Test
+    fun testUnaryMinus() = runTest{
+        eval("""
+            assertEquals( -1 * π, 0 - π )
+            assertEquals( -1 * π, -π )
+            """.trimIndent())
+    }
+
+    @Test
+    fun testSimpleTypes() = runTest{
+        testScope().eval("""
+            testEncode(null)
+            testEncode(0)
+            testEncode(47)
+            testEncode(-21)
+            testEncode(true)
+            testEncode(false)
+            testEncode(1.22345)
+            testEncode(-π)
+
+            import lyng.time
+            testEncode(Instant.now().truncateToSecond())
+            testEncode(Instant.now().truncateToMillisecond())
+            testEncode(Instant.now().truncateToMicrosecond())
+
+            testEncode("Hello, world".encodeUtf8())
+            testEncode("Hello, world")
+            
+        """.trimIndent())
+    }
+
+    @Test
+    fun testLzw() {
+        // Example usage
+//        val original = "TOBEORNOTTOBEORTOBEORNOT"
+//        println("Original: $original")
+        println("Length: ${original.length}")
+
+        // Compress
+        val out = MemoryBitOutput()
+        LZW.compress(original.encodeToByteArray().toUByteArray(), out)
+//        println("\nCompressed codes: ${out.toUByteArray().toDump()}")
+        println("Number of codes: ${out.toBitArray().bytesSize}")
+        println("Copression rate: ${out.toBitArray().bytesSize.toDouble() / original.length.toDouble()}")
+//        // Decompress
+        val decompressed = LZW.decompress(MemoryBitInput(out), original.length).toByteArray().decodeToString()
+//        println("\nDecompressed: $decompressed")
+        println("Length: ${decompressed.length}")
+
+        // Verification
+        println("\nOriginal and decompressed match: ${original == decompressed}")
+        assertEquals(original, decompressed)
+    }
+
+    @Test
+    fun testTinyBits() {
+        var a0 = TinyBits()
+
+        assertEquals(a0, a0)
+        a0 = a0.insertBit(0)
+        a0 = a0.insertBit(1)
+        a0 = a0.insertBit(1)
+        a0 = a0.insertBit(1)
+        a0 = a0.insertBit(0)
+        a0 = a0.insertBit(1)
+//        println(a0)
+        assertEquals("011101", a0.toString())
+        val bin = MemoryBitInput(MemoryBitOutput().apply { putBits(a0) })
+        var result = TinyBits()
+        for( i in a0.indices)  result = result.insertBit(bin.getBit())
+        assertEquals(a0, result)
+    }
+
+    @Test
+    fun testHuffman() {
+        val x = original.encodeToByteArray().toUByteArray()
+//        val x ="hello, world!".toByteArray().asUByteArray()// original.encodeToByteArray().toUByteArray()
+        println("Original : ${x.size}")
+        val lzw = LZW.compress(x).bytes
+        println("LZW      : ${lzw.size}")
+        val ba = Huffman.compress(x, Huffman.byteAlphabet)
+        val huff = ba.bytes
+        println("Huffman  : ${huff.size}")
+        val lzwhuff = Huffman.compress(lzw, Huffman.byteAlphabet).bytes
+        println("LZW+HUFF : ${lzwhuff.size}")
+        val compressed = Huffman.compress(x,Huffman.byteAlphabet)
+        val decompressed = Huffman.decompress(compressed.toBitInput(),Huffman.byteAlphabet)
+        assertContentEquals(x, decompressed)
+    }
+
+    @Test
+    fun testGenerateCanonicalHuffmanCodes() {
+        val frequencies = LynonType.entries.map { it.defaultFrequency }.toTypedArray()
+        val alphabet = object : Huffman.Alphabet<LynonType> {
+            override val maxOrdinal = LynonType.entries.size
+
+//            val bitSize = sizeInBits(maxOrdinal)
+
+            override fun decodeOrdinalTo(bout: BitOutput, ordinal: Int) {
+                TODO("Not yet implemented")
+            }
+
+            override fun get(ordinal: Int): LynonType {
+                TODO("Not yet implemented")
+            }
+
+            override fun ordinalOf(value: LynonType): Int = value.ordinal
+        }
+        for(code in Huffman.generateCanonicalCodes(frequencies, alphabet)) {
+            println("${code?.bits}: ${code?.ordinal?.let { LynonType.entries[it] }}")
+        }
+    }
+
+    @Test
+    fun testBitListSmall() {
+        var t = TinyBits()
+        for( i in listOf(1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1) )
+            t = t.insertBit(i)
+        assertEquals(1, t[0])
+        assertEquals(1, t[1])
+        assertEquals(0, t[2])
+        assertEquals("1101000111101",t.toString())
+        t[0] = 0
+        t[1] = 0
+        t[2] = 1
+        assertEquals("0011000111101",t.toString())
+        t[12] = 0
+        t[11] = 1
+        assertEquals("0011000111110",t.toString())
+    }
+
+    @Test
+    fun testBitListSerialization() {
+        // this also tests bitArray with first and last bytes
+        val bout = MemoryBitOutput()
+        assertEquals("1101", bitListOf(1, 1, 0, 1).toString())
+        bout.putBits(bitListOf(1, 1, 0, 1))
+        bout.putBits(bitListOf( 0, 0))
+        bout.putBits(bitListOf( 0, 1, 1, 1, 1, 0, 1))
+        val x = bout.toBitArray()
+        assertEquals("1101000111101",x.toString())
+    }
+
+
+    @Test
+    fun testCompressionWithOffsets() {
+        val src = "to be or not to be or not to be or not to be or not to be"
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(1571UL)
+        LZW.compress(src.encodeToByteArray(), bout)
+        bout.packUnsigned(157108UL)
+        val bin = bout.toBitInput()
+        assertEquals(1571UL, bin.unpackUnsigned())
+        assertEquals(src, LZW.decompress(bin, src.length).asByteArray().decodeToString())
+        assertEquals(157108UL, bin.unpackUnsigned())
+    }
+
+    @Test
+    fun testCompressionRecord() {
+        val bout = MemoryBitOutput()
+        val src = "to be or not to be or not to be or not to be or not to be"
+        val src2 = "to be or not to be"
+        val src3 = "ababababab"
+        bout.compress(src)
+        bout.compress(src2)
+        bout.compress(src3)
+        val bin = bout.toBitInput()
+        assertEquals(src, bin.decompressString())
+        assertEquals(src2, bin.decompressString())
+        assertEquals(src3, bin.decompressString())
+    }
+
+}
+

lynglib/src/jvmTest/kotlin/OtherTests.kt

@@ -0,0 +1,53 @@
+import junit.framework.TestCase.assertEquals
+import kotlinx.coroutines.runBlocking
+import net.sergeych.lyng.ModuleScope
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.eval
+import net.sergeych.lyng.pacman.InlineSourcesImportProvider
+import net.sergeych.lyng.toSource
+import kotlin.test.Test
+
+class OtherTests {
+    @Test
+    fun testImports3() = runBlocking {
+        val foosrc = """
+            package lyng.foo
+            
+            import lyng.bar            
+            
+            fun foo() { "foo1" }
+            """.trimIndent()
+        val barsrc = """
+            package lyng.bar
+            
+            fun bar() { "bar1" }
+            """.trimIndent()
+        val pm = InlineSourcesImportProvider(
+            listOf(
+            Source("foosrc", foosrc),
+            Source("barsrc", barsrc),
+        ))
+
+        val src = """
+            import lyng.foo
+            
+            foo() + " / " + bar()
+            """.trimIndent().toSource("test")
+
+        val scope = ModuleScope(pm, src)
+        assertEquals("foo1 / bar1", scope.eval(src).toString())
+    }
+
+    @Test
+    fun testInstantTruncation() = runBlocking {
+        eval("""
+            import lyng.time
+            val t1 = Instant()
+            val t2 = Instant()
+//            assert( t1 != t2 )
+            println(t1 - t2)
+        """.trimIndent())
+        Unit
+    }
+
+}