refs #35 serializatino framework refactored: implementation put in open methods of Obj/ObjClass for flexibility

docs/OOP.md

@@ -250,11 +250,10 @@ Note `Real` class: it is global variable for Real class; there are such class in
     assert('$'::class == Char)
     >>> void
 
-More complex is singleton classes, because you don't need to compare their class
+Singleton classes also have class:
-instances and generally don't need them at all, these are normally just Obj:
 
     null::class
-    >>> Obj
+    >>> Null
 
 At this time, `Obj` can't be accessed as a class.
 

docs/time.md

@@ -98,11 +98,16 @@ so it is possible to truncate it to milliseconds, microseconds or seconds:
     import lyng.serialization
 
     // max supported size (now microseconds for serialized value):
-    assert( Lynon.encode(Instant.now()).size in [8,9] )
+    // note that encoding return _bit array_ and this is a _bit size_:
+    val s0 = Lynon.encode(Instant.now()).size
+
     // shorter: milliseconds only
-    assertEquals( 7, Lynon.encode(Instant.now().truncateToMillisecond()).size )
+    val s1 = Lynon.encode(Instant.now().truncateToMillisecond()).size
+
     // truncated to seconds, good for file mtime, etc:
-    assertEquals( 6, Lynon.encode(Instant.now().truncateToSecond()).size )
+    val s2 = Lynon.encode(Instant.now().truncateToSecond()).size 
+    assert( s1 < s0 )
+    assert( s2 < s1 )
     >>> void
 
 ## Formatting instants

docs/tutorial.md

@@ -1217,25 +1217,26 @@ Concatenation is a `+`: `"hello " + name` works as expected. No confusion.
 
 Typical set of String functions includes:
 
-| fun/prop           | description / notes                                        |
+| fun/prop          | description / notes                                        |
-|--------------------|------------------------------------------------------------|
+|-------------------|------------------------------------------------------------|
-| lower()            | change case to unicode upper                               |
+| lower()           | change case to unicode upper                               |
-| upper()            | change case to unicode lower                               |
+| upper()           | change case to unicode lower                               |
 | startsWith(prefix) | true if starts with a prefix                               |
-| endsWith(prefix)   | true if ends with a prefix                                 |
+| endsWith(prefix)  | true if ends with a prefix                                 |
-| take(n)            | get a new string from up to n first characters             |
+| take(n)           | get a new string from up to n first characters             |
-| takeLast(n)        | get a new string from up to n last characters              |
+| takeLast(n)       | get a new string from up to n last characters              |
-| drop(n)            | get a new string dropping n first chars, or empty string   |
+| drop(n)           | get a new string dropping n first chars, or empty string   |
-| dropLast(n)        | get a new string dropping n last chars, or empty string    |
+| dropLast(n)       | get a new string dropping n last chars, or empty string    |
-| size               | size in characters like `length` because String is [Array] |
+| size              | size in characters like `length` because String is [Array] |
-| (args...)          | sprintf-like formatting, see [string formatting]           |
+| (args...)         | sprintf-like formatting, see [string formatting]           |
-| [index]            | character at index                                         | 
+| [index]           | character at index                                         | 
-| [Range]            | substring at range                                         |
+| [Range]           | substring at range                                         |
-| s1 + s2            | concatenation                                              |
+| s1 + s2           | concatenation                                              |
-| s1 += s2           | self-modifying concatenation                               |
+| s1 += s2          | self-modifying concatenation                               |
-| toReal()           | attempts to parse string as a Real value                   |
+| toReal()          | attempts to parse string as a Real value                   |
-| toInt()            | parse string to Int value                                  |
+| toInt()           | parse string to Int value                                  |
-| characters()       | create [List] of characters (1)                            |
+| characters()      | create [List] of characters (1)                            |
+| encodeUtf8()      | returns [Buffer] with characters encoded to utf8           |
 
 (1)
 : List is mutable therefore a new copy is created on each call.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -46,6 +46,10 @@ open class Scope(
     fun raiseIllegalArgument(message: String = "Illegal argument error"): Nothing =
         raiseError(ObjIllegalArgumentException(this, message))
 
+    @Suppress("unused")
+    fun raiseIllegalState(message: String = "Illegal argument error"): Nothing =
+        raiseError(ObjIllegalStateException(this, message))
+
     @Suppress("unused")
     fun raiseNoSuchElement(message: String = "No such element"): Nothing =
         raiseError(ObjIllegalArgumentException(this, message))

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Accessor.kt

@@ -0,0 +1,33 @@
+
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Compiler
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.ScriptError
+
+// avoid KDOC bug: keep it
+@Suppress("unused")
+typealias DocCompiler = Compiler
+/**
+ * When we need read-write access to an object in some abstract storage, we need Accessor,
+ * as in-site assigning is not always sufficient, in general case we need to replace the object
+ * in the storage.
+ *
+ * Note that assigning new value is more complex than just replacing the object, see how assignment
+ * operator is implemented in [Compiler.allOps].
+ */
+data class Accessor(
+    val getter: suspend (Scope) -> ObjRecord,
+    val setterOrNull: (suspend (Scope, Obj) -> Unit)?
+) {
+    /**
+     * Simplified constructor for immutable stores.
+     */
+    constructor(getter: suspend (Scope) -> ObjRecord) : this(getter, null)
+
+    /**
+     * Get the setter or throw.
+     */
+    fun setter(pos: Pos) = setterOrNull ?: throw ScriptError(pos, "can't assign value")
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt

@@ -6,54 +6,19 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import net.sergeych.bintools.encodeToHex
 import net.sergeych.lyng.*
+import net.sergeych.lynon.LynonDecoder
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import net.sergeych.synctools.ProtectedOp
 import net.sergeych.synctools.withLock
 import kotlin.contracts.ExperimentalContracts
 
-/**
- * Record to store object with access rules, e.g. [isMutable] and access level [visibility].
- */
-data class ObjRecord(
-    var value: Obj,
-    val isMutable: Boolean,
-    val visibility: Visibility = Visibility.Public,
-    var importedFrom: Scope? = null
-) {
-    @Suppress("unused")
-    fun qualifiedName(name: String): String =
-        "${importedFrom?.packageName ?: "anonymous"}.$name"
-}
-
-/**
- * When we need read-write access to an object in some abstract storage, we need Accessor,
- * as in-site assigning is not always sufficient, in general case we need to replace the object
- * in the storage.
- *
- * Note that assigning new value is more complex than just replacing the object, see how assignment
- * operator is implemented in [Compiler.allOps].
- */
-data class Accessor(
-    val getter: suspend (Scope) -> ObjRecord,
-    val setterOrNull: (suspend (Scope, Obj) -> Unit)?
-) {
-    /**
-     * Simplified constructor for immutable stores.
-     */
-    constructor(getter: suspend (Scope) -> ObjRecord) : this(getter, null)
-
-    /**
-     * Get the setter or throw.
-     */
-    fun setter(pos: Pos) = setterOrNull ?: throw ScriptError(pos, "can't assign value")
-}
-
 open class Obj {
 
     open val isConst: Boolean = false
 
     fun ensureNotConst(scope: Scope) {
-        if( isConst ) scope.raiseError("can't assign to constant")
+        if (isConst) scope.raiseError("can't assign to constant")
     }
 
     val isNull by lazy { this === ObjNull }
@@ -273,33 +238,35 @@ open class Obj {
     val asReadonly: ObjRecord by lazy { ObjRecord(this, false) }
     val asMutable: ObjRecord by lazy { ObjRecord(this, true) }
 
-    open suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    open suspend fun lynonType(): LynonType = LynonType.Other
+
+    open suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
         scope.raiseNotImplemented()
     }
 
     companion object {
 
         val rootObjectType = ObjClass("Obj").apply {
-                addFn("toString") {
+            addFn("toString") {
-                    thisObj.asStr
+                thisObj.asStr
-                }
+            }
-                addFn("contains") {
+            addFn("contains") {
-                    ObjBool(thisObj.contains(this, args.firstAndOnly()))
+                ObjBool(thisObj.contains(this, args.firstAndOnly()))
-                }
+            }
-                // utilities
+            // utilities
-                addFn("let") {
+            addFn("let") {
-                    args.firstAndOnly().callOn(copy(Arguments(thisObj)))
+                args.firstAndOnly().callOn(copy(Arguments(thisObj)))
-                }
+            }
-                addFn("apply") {
+            addFn("apply") {
-                    val newContext = ( thisObj as? ObjInstance)?.instanceScope ?: this
+                val newContext = (thisObj as? ObjInstance)?.instanceScope ?: this
-                    args.firstAndOnly()
+                args.firstAndOnly()
-                        .callOn(newContext)
+                    .callOn(newContext)
-                    thisObj
+                thisObj
-                }
+            }
-                addFn("also") {
+            addFn("also") {
-                    args.firstAndOnly().callOn(copy(Arguments(thisObj)))
+                args.firstAndOnly().callOn(copy(Arguments(thisObj)))
-                    thisObj
+                thisObj
-                }
+            }
             addFn("getAt") {
                 requireExactCount(1)
                 thisObj.getAt(this, requiredArg<Obj>(0))
@@ -395,6 +362,26 @@ object ObjNull : Obj() {
     override suspend fun toKotlin(scope: Scope): Any? {
         return null
     }
+
+    override suspend fun lynonType(): LynonType {
+        return LynonType.Null
+    }
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        if (lynonType == null) {
+            encoder.putBit(0)
+        }
+    }
+
+    override val objClass: ObjClass by lazy {
+        object : ObjClass("Null") {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj {
+                if (lynonType == LynonType.Null)
+                    return this@ObjNull
+                else
+                    scope.raiseIllegalState("can't deserialize null directly or with wrong type: ${lynonType}")
+            }
+        }
+    }
 }
 
 interface Numeric {
@@ -525,6 +512,9 @@ class ObjIndexOutOfBoundsException(scope: Scope, message: String = "index out of
 class ObjIllegalArgumentException(scope: Scope, message: String = "illegal argument") :
     ObjException("IllegalArgumentException", scope, message)
 
+class ObjIllegalStateException(scope: Scope, message: String = "illegal state") :
+    ObjException("IllegalStateException", scope, message)
+
 @Suppress("unused")
 class ObjNoSuchElementException(scope: Scope, message: String = "no such element") :
     ObjException("IllegalArgumentException", scope, message)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBitBuffer.kt

@@ -0,0 +1,37 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.bintools.toDump
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.BitArray
+
+class ObjBitBuffer(val bitArray: BitArray) : Obj() {
+
+    override val objClass = type
+
+    override suspend fun getAt(scope: Scope, index: Obj): Obj {
+        return bitArray[index.toLong()].toObj()
+    }
+
+    companion object {
+        val type = object: ObjClass("BitBuffer", ObjArray) {
+
+        }.apply {
+            addFn("toBuffer") {
+                requireNoArgs()
+                ObjBuffer(thisAs<ObjBitBuffer>().bitArray.asUbyteArray())
+            }
+            addFn("toDump") {
+                requireNoArgs()
+                ObjString(
+                    thisAs<ObjBitBuffer>().bitArray.asUbyteArray().toDump()
+                )
+            }
+            addFn("size") {
+                thisAs<ObjBitBuffer>().bitArray.size.toObj()
+            }
+            addFn("sizeInBytes") {
+                ObjInt((thisAs<ObjBitBuffer>().bitArray.size + 7) shr 3)
+            }
+        }
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBool.kt

@@ -3,6 +3,7 @@ package net.sergeych.lyng.obj
 import net.sergeych.lyng.Scope
 import net.sergeych.lynon.LynonDecoder
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 
 data class ObjBool(val value: Boolean) : Obj() {
     override val asStr by lazy { ObjString(value.toString()) }
@@ -30,7 +31,9 @@ data class ObjBool(val value: Boolean) : Obj() {
         return value
     }
 
-    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    override suspend fun lynonType(): LynonType = LynonType.Bool
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
         encoder.encodeBoolean(value)
     }
 
@@ -45,7 +48,7 @@ data class ObjBool(val value: Boolean) : Obj() {
 
     companion object {
         val type = object : ObjClass("Bool") {
-            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj {
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder,lynonType: LynonType?): Obj {
                 return ObjBool(decoder.unpackBoolean())
             }
         }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt

@@ -2,9 +2,13 @@ package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
+import net.sergeych.bintools.encodeToHex
 import net.sergeych.bintools.toDump
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import kotlin.math.min
 
 open class ObjBuffer(val byteArray: UByteArray) : Obj() {
@@ -63,7 +67,7 @@ open class ObjBuffer(val byteArray: UByteArray) : Obj() {
     }
 
     override fun toString(): String {
-        return "Buffer(${byteArray.toList()})"
+        return "Buffer(${byteArray.encodeToHex()})"
     }
 
     override fun equals(other: Any?): Boolean {
@@ -75,6 +79,14 @@ open class ObjBuffer(val byteArray: UByteArray) : Obj() {
         return byteArray contentEquals other.byteArray
     }
 
+    override suspend fun lynonType(): LynonType = LynonType.Buffer
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.encodeCached(byteArray) {
+            bout.compress(byteArray.asByteArray())
+        }
+    }
+
     companion object {
         private suspend fun createBufferFrom(scope: Scope, obj: Obj): ObjBuffer =
             when (obj) {
@@ -124,6 +136,12 @@ open class ObjBuffer(val byteArray: UByteArray) : Obj() {
                     }
                 }
             }
+
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
+                ObjBuffer( decoder.decodeCached {
+                    decoder.decompress().asUByteArray()
+                })
+
         }.apply {
             createField("size",
                 statement {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjClass.kt

@@ -2,6 +2,7 @@ package net.sergeych.lyng.obj
 
 import net.sergeych.lyng.*
 import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonType
 
 val ObjClassType by lazy { ObjClass("Class") }
 
@@ -98,7 +99,7 @@ open class ObjClass(
         return classMembers[name]?.value?.invoke(scope, this, args) ?: super.invokeInstanceMethod(scope, name, args)
     }
 
-    open fun deserialize(scope: Scope, decoder: LynonDecoder): Obj = scope.raiseNotImplemented()
+    open suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj = scope.raiseNotImplemented()
 }
 
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstance.kt

@@ -3,6 +3,7 @@ package net.sergeych.lyng.obj
 import net.sergeych.lyng.Arguments
 import net.sergeych.lyng.Scope
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 
 class ObjInstance(override val objClass: ObjClass) : Obj() {
 
@@ -45,7 +46,7 @@ class ObjInstance(override val objClass: ObjClass) : Obj() {
         return "${objClass.className}($fields)"
     }
 
-    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
         val meta = objClass.constructorMeta
             ?: scope.raiseError("can't serialize non-serializable object (no constructor meta)")
         for( p in meta.params) {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstant.kt

@@ -5,7 +5,10 @@ import kotlinx.datetime.Instant
 import kotlinx.datetime.isDistantFuture
 import kotlinx.datetime.isDistantPast
 import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
 import net.sergeych.lynon.LynonSettings
+import net.sergeych.lynon.LynonType
 
 class ObjInstant(val instant: Instant,val truncateMode: LynonSettings.InstantTruncateMode=LynonSettings.InstantTruncateMode.Microsecond) : Obj() {
     override val objClass: ObjClass get() = type
@@ -53,6 +56,22 @@ class ObjInstant(val instant: Instant,val truncateMode: LynonSettings.InstantTru
         return instant == other.instant
     }
 
+    override suspend fun lynonType(): LynonType = LynonType.Instant
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        encoder.putBits(truncateMode.ordinal, 2)
+        when(truncateMode) {
+            LynonSettings.InstantTruncateMode.Millisecond ->
+                encoder.encodeSigned(instant.toEpochMilliseconds())
+            LynonSettings.InstantTruncateMode.Second ->
+                encoder.encodeSigned(instant.epochSeconds)
+            LynonSettings.InstantTruncateMode.Microsecond -> {
+                encoder.encodeSigned(instant.epochSeconds)
+                encoder.encodeUnsigned(instant.nanosecondsOfSecond.toULong() / 1000UL)
+            }
+        }
+    }
+
     companion object {
         val distantFuture by lazy {
             ObjInstant(Instant.DISTANT_FUTURE)
@@ -86,6 +105,26 @@ class ObjInstant(val instant: Instant,val truncateMode: LynonSettings.InstantTru
                     }
                 )
             }
+
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj {
+                val mode = LynonSettings.InstantTruncateMode.entries[decoder.getBitsAsInt(2)]
+                return when (mode) {
+                    LynonSettings.InstantTruncateMode.Microsecond -> ObjInstant(
+                        Instant.fromEpochSeconds(
+                            decoder.unpackSigned(), decoder.unpackUnsignedInt() * 1000
+                        )
+                    )
+                    LynonSettings.InstantTruncateMode.Millisecond -> ObjInstant(
+                        Instant.fromEpochMilliseconds(
+                            decoder.unpackSigned()
+                        )
+                    )
+                    LynonSettings.InstantTruncateMode.Second -> ObjInstant(
+                        Instant.fromEpochSeconds(decoder.unpackSigned())
+                    )
+                }
+            }
+
         }.apply {
             addFn("epochSeconds") {
                 val instant = thisAs<ObjInstant>().instant

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt

@@ -3,8 +3,9 @@ package net.sergeych.lyng.obj
 import net.sergeych.lyng.Scope
 import net.sergeych.lynon.LynonDecoder
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 
-class ObjInt(var value: Long,override val isConst: Boolean = false) : Obj(), Numeric {
+class ObjInt(var value: Long, override val isConst: Boolean = false) : Obj(), Numeric {
     override val asStr get() = ObjString(value.toString())
     override val longValue get() = value
     override val doubleValue get() = value.toDouble()
@@ -101,16 +102,37 @@ class ObjInt(var value: Long,override val isConst: Boolean = false) : Obj(), Num
         return ObjInt(-value)
     }
 
-    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    override suspend fun lynonType(): LynonType = when (value) {
-        encoder.encodeSigned(value)
+        0L -> LynonType.Int0
+        else -> {
+            if (value > 0) LynonType.IntPositive
+            else LynonType.IntNegative
+        }
+    }
+
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
+        when (lynonType) {
+            null -> encoder.encodeSigned(value)
+            LynonType.Int0 -> {}
+            LynonType.IntPositive -> encoder.encodeUnsigned(value.toULong())
+            LynonType.IntNegative -> encoder.encodeUnsigned((-value).toULong())
+            else -> scope.raiseIllegalArgument("Unsupported lynon type code for Int: $lynonType")
+        }
     }
 
     companion object {
         val Zero = ObjInt(0, true)
         val One = ObjInt(1, true)
-        val type = object: ObjClass("Int") {
+        val type = object : ObjClass("Int") {
-            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
-                ObjInt(decoder.unpackSigned())
+                when (lynonType) {
+                    null -> ObjInt(decoder.unpackSigned())
+                    LynonType.Int0 -> Zero
+                    LynonType.IntPositive -> ObjInt(decoder.unpackUnsigned().toLong())
+                    LynonType.IntNegative -> ObjInt(-decoder.unpackUnsigned().toLong())
+                    else -> scope.raiseIllegalState("illegal type code for Int: $lynonType")
+                }
         }
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjReal.kt

@@ -5,6 +5,7 @@ import net.sergeych.lyng.Scope
 import net.sergeych.lyng.statement
 import net.sergeych.lynon.LynonDecoder
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import kotlin.math.floor
 import kotlin.math.roundToLong
 
@@ -65,13 +66,15 @@ data class ObjReal(val value: Double) : Obj(), Numeric {
         return ObjReal(-value)
     }
 
-    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    override suspend fun lynonType(): LynonType = LynonType.Real
+
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
         encoder.encodeReal(value)
     }
 
     companion object {
         val type: ObjClass = object : ObjClass("Real") {
-            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
                 ObjReal(decoder.unpackDouble())
         }.apply {
             createField(

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRecord.kt

@@ -0,0 +1,18 @@
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Visibility
+
+/**
+ * Record to store object with access rules, e.g. [isMutable] and access level [visibility].
+ */
+data class ObjRecord(
+    var value: Obj,
+    val isMutable: Boolean,
+    val visibility: Visibility = Visibility.Public,
+    var importedFrom: Scope? = null
+) {
+    @Suppress("unused")
+    fun qualifiedName(name: String): String =
+        "${importedFrom?.packageName ?: "anonymous"}.$name"
+}

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt

@@ -6,6 +6,7 @@ import net.sergeych.lyng.Scope
 import net.sergeych.lyng.statement
 import net.sergeych.lynon.LynonDecoder
 import net.sergeych.lynon.LynonEncoder
+import net.sergeych.lynon.LynonType
 import net.sergeych.sprintf.sprintf
 
 @Serializable
@@ -80,13 +81,13 @@ data class ObjString(val value: String) : Obj() {
         return value == other.value
     }
 
-    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) {
         encoder.encodeBinaryData(value.encodeToByteArray())
     }
 
     companion object {
         val type = object : ObjClass("String") {
-            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+            override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj =
                 ObjString(
                     decoder.unpackBinaryData().decodeToString()
 //                        ?: scope.raiseError("unexpected end of data")
@@ -135,6 +136,7 @@ data class ObjString(val value: String) : Obj() {
                     thisAs<ObjString>().value.map { ObjChar(it) }.toMutableList()
                 )
             }
+            addFn("encodeUtf8") { ObjBuffer(thisAs<ObjString>().value.encodeToByteArray().asUByteArray()) }
             addFn("size") { ObjInt(thisAs<ObjString>().value.length.toLong()) }
             addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble()) }
         }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -1,14 +1,23 @@
 package net.sergeych.lynon
 
-import kotlinx.datetime.Instant
 import net.sergeych.lyng.Scope
-import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjClass
 
 open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSettings.default) {
 
-    val cache = mutableListOf<Obj>()
+    fun getBitsAsInt(bitsSize: Int): Int {
+        return bin.getBits(bitsSize).toInt()
+    }
 
-    inline fun decodeCached(f: LynonDecoder.() -> Obj): Obj {
+    fun unpackUnsignedInt(): Int = bin.unpackUnsigned().toInt()
+
+    fun decompress() = bin.decompress()
+
+    val cache = mutableListOf<Any>()
+
+
+    inline fun <T : Any>decodeCached(f: LynonDecoder.() -> T): T {
         return if (bin.getBit() == 0) {
             // unpack and cache
             f().also {
@@ -20,46 +29,19 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe
             val id = bin.getBitsOrNull(size)?.toInt()
                 ?: throw RuntimeException("Invalid object id: unexpected end of stream")
             if (id >= cache.size) throw RuntimeException("Invalid object id: $id should be in 0..<${cache.size}")
-            cache[id]
+            @Suppress("UNCHECKED_CAST")
+            cache[id] as T
         }
     }
 
-    fun decodeAny(scope: Scope): Obj = decodeCached {
+    suspend fun decodeAny(scope: Scope): Obj = decodeCached {
         val type = LynonType.entries[bin.getBits(4).toInt()]
-        return when (type) {
+        type.objClass.deserialize(scope, this, type)
-            LynonType.Null -> ObjNull
-            LynonType.Int0 -> ObjInt.Zero
-            LynonType.IntPositive -> ObjInt(bin.unpackUnsigned().toLong())
-            LynonType.IntNegative -> ObjInt(-bin.unpackUnsigned().toLong())
-            LynonType.Bool -> ObjBool(bin.getBit() == 1)
-            LynonType.Real -> ObjReal(bin.unpackDouble())
-            LynonType.Instant -> {
-                val mode = LynonSettings.InstantTruncateMode.entries[bin.getBits(2).toInt()]
-                when (mode) {
-                    LynonSettings.InstantTruncateMode.Microsecond -> ObjInstant(
-                        Instant.fromEpochSeconds(
-                            bin.unpackSigned(), bin.unpackUnsigned().toInt() * 1000
-                        )
-                    )
-                    LynonSettings.InstantTruncateMode.Millisecond -> ObjInstant(
-                        Instant.fromEpochMilliseconds(
-                            bin.unpackSigned()
-                        )
-                    )
-                    LynonSettings.InstantTruncateMode.Second -> ObjInstant(
-                        Instant.fromEpochSeconds(bin.unpackSigned())
-                    )
-                }
-            }
-
-            else -> {
-                scope.raiseNotImplemented("lynon type $type")
-            }
-        }
     }
 
-    fun unpackObject(scope: Scope, type: ObjClass): Obj {
+    // todo: rewrite/remove?
-        return decodeCached { type.deserialize(scope, this) }
+    suspend fun unpackObject(scope: Scope, type: ObjClass): Obj {
+        return decodeCached { type.deserialize(scope, this, null) }
     }
 
     fun unpackBinaryData(): ByteArray = bin.decompress()
@@ -79,4 +61,8 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe
         return bin.unpackSigned()
     }
 
+    fun unpackUnsigned(): ULong {
+        return bin.unpackUnsigned()
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -1,30 +1,31 @@
 package net.sergeych.lynon
 
+import net.sergeych.bintools.ByteChunk
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.*
 
-enum class LynonType {
+enum class LynonType(val objClass: ObjClass) {
-    Null,
+    Null(ObjNull.objClass),
-    Int0,
+    Int0(ObjInt.type),
-    IntNegative,
+    IntNegative(ObjInt.type),
-    IntPositive,
+    IntPositive(ObjInt.type),
-    String,
+    String(ObjString.type),
-    Real,
+    Real(ObjReal.type),
-    Bool,
+    Bool(ObjBool.type),
-    List,
+    List(ObjList.type),
-    Map,
+    Map(ObjMap.type),
-    Set,
+    Set(ObjSet.type),
-    Buffer,
+    Buffer(ObjBuffer.type),
-    Instant,
+    Instant(ObjInstant.type),
-    Duration,
+    Duration(ObjDuration.type),
-    Other;
+    Other(Obj.rootObjectType);
 }
 
 open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonSettings.default) {
 
     val cache = mutableMapOf<Any, Int>()
 
-    private suspend fun encodeCached(item: Any, packer: suspend LynonEncoder.() -> Unit) {
+    suspend fun encodeCached(item: Any, packer: suspend LynonEncoder.() -> Unit) {
 
         suspend fun serializeAndCache(key: Any=item) {
             bout.putBit(0)
@@ -40,7 +41,8 @@ open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonS
                 bout.putBits(cacheId.toULong(), size)
             } ?: serializeAndCache()
 
-            is ByteArray, is UByteArray ->  serializeAndCache()
+            is ByteArray -> serializeAndCache(ByteChunk(item.asUByteArray()))
+            is UByteArray ->  serializeAndCache(ByteChunk(item))
         }
     }
 
@@ -52,48 +54,9 @@ open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonS
      */
     suspend fun encodeAny(scope: Scope,value: Obj) {
         encodeCached(value) {
-            when(value) {
+            val type = value.lynonType()
-                is ObjNull -> putType(LynonType.Null)
+            putType(type)
-                is ObjInt -> {
+            value.serialize(scope, this, type)
-                    when {
-                        value.value == 0L -> putType(LynonType.Int0)
-                        value.value < 0 -> {
-                            putType(LynonType.IntNegative)
-                            encodeUnsigned((-value.value).toULong())
-                        }
-                        else -> {
-                            putType(LynonType.IntPositive)
-                            encodeUnsigned(value.value.toULong())
-                        }
-                    }
-                }
-                is ObjBool -> {
-                    putType(LynonType.Bool)
-                    encodeBoolean(value.value)
-                }
-                is ObjReal -> {
-                    putType(LynonType.Real)
-                    encodeReal(value.value)
-                }
-                is ObjInstant -> {
-                    putType(LynonType.Instant)
-                    bout.putBits(value.truncateMode.ordinal, 2)
-                    // todo: favor truncation mode from ObjInstant
-                    when(value.truncateMode) {
-                        LynonSettings.InstantTruncateMode.Millisecond ->
-                            encodeSigned(value.instant.toEpochMilliseconds())
-                        LynonSettings.InstantTruncateMode.Second ->
-                            encodeSigned(value.instant.epochSeconds)
-                        LynonSettings.InstantTruncateMode.Microsecond -> {
-                            encodeSigned(value.instant.epochSeconds)
-                            encodeUnsigned(value.instant.nanosecondsOfSecond.toULong() / 1000UL)
-                        }
-                    }
-                }
-                else -> {
-                    TODO()
-                }
-            }
         }
     }
 
@@ -103,7 +66,7 @@ open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonS
 
     suspend fun encodeObj(scope: Scope, obj: Obj) {
         encodeCached(obj) {
-            obj.serialize(scope, this)
+            obj.serialize(scope, this, null)
         }
     }
 
@@ -133,4 +96,12 @@ open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonS
         bout.putBit(if (value) 1 else 0)
     }
 
+    fun putBits(value: Int, sizeInBits: Int) {
+        bout.putBits(value.toULong(), sizeInBits)
+    }
+
+    fun putBit(bit: Int) {
+        bout.putBit(bit)
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/packer.kt

@@ -2,7 +2,7 @@ package net.sergeych.lynon
 
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.Obj
-import net.sergeych.lyng.obj.ObjBuffer
+import net.sergeych.lyng.obj.ObjBitBuffer
 import net.sergeych.lyng.obj.ObjClass
 import net.sergeych.lyng.obj.ObjString
 
@@ -15,11 +15,12 @@ val ObjLynonClass = object : ObjClass("Lynon") {
         val bout = MemoryBitOutput()
         val serializer = LynonEncoder(bout)
         serializer.encodeAny(this, obj)
-        return ObjBuffer(bout.toBitArray().bytes)
+        return ObjBitBuffer(bout.toBitArray())
     }
 
-    suspend fun Scope.decodeAny(buffer: ObjBuffer): Obj {
+    suspend fun Scope.decodeAny(source: Obj): Obj {
-        val bin = BitArray(buffer.byteArray,8).toInput()
+        if( source !is ObjBitBuffer) throw Exception("Invalid source: $source")
+        val bin = source.bitArray.toInput()
         val deserializer = LynonDecoder(bin)
         return deserializer.decodeAny(this)
     }
@@ -30,6 +31,6 @@ val ObjLynonClass = object : ObjClass("Lynon") {
         encodeAny(requireOnlyArg<Obj>())
     }
     addClassFn("decode") {
-        decodeAny(requireOnlyArg<ObjBuffer>())
+        decodeAny(requireOnlyArg<Obj>())
     }
 }

lynglib/src/jvmTest/kotlin/LynonTests.kt

@@ -309,9 +309,9 @@ class LynonTests {
             assertEquals( -1 * π, -π )
             """.trimIndent())
     }
-    
+
     @Test
-    fun testIntsNulls() = runTest{
+    fun testSimpleTypes() = runTest{
         testScope().eval("""
             testEncode(null)
             testEncode(0)
@@ -326,6 +326,9 @@ class LynonTests {
             testEncode(Instant.now().truncateToSecond())
             testEncode(Instant.now().truncateToMillisecond())
             testEncode(Instant.now().truncateToMicrosecond())
+            
+            testEncode("Hello, world".encodeUtf8())
+            
         """.trimIndent())
     }