refs #35 minimal serialization of simple types in bit-effective format

lyng/src/commonMain/kotlin/Common.kt

@@ -9,7 +9,11 @@ import com.github.ajalt.clikt.parameters.arguments.optional
 import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import kotlinx.coroutines.runBlocking
-import net.sergeych.lyng.*
+import net.sergeych.lyng.LyngVersion
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.ScriptError
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.obj.*
 import okio.FileSystem
 import okio.Path.Companion.toPath
 import okio.SYSTEM

lynglib/src/commonMain/kotlin/net/sergeych/lyng/AppliedScope.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.ObjRecord
+
 /**
  * Special version of the [Scope] used to `apply` new this object to
  * _parent context property.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ArgsDeclaration.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjList
+
 /**
  * List of argument declarations in the __definition__ of the lambda, class constructor,
  * function, etc. It is created by [Compiler.parseArgsDeclaration]

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Arguments.kt

@@ -1,5 +1,9 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjIterable
+import net.sergeych.lyng.obj.ObjList
+
 data class ParsedArgument(val value: Statement, val pos: Pos, val isSplat: Boolean = false)
 
 suspend fun Collection<ParsedArgument>.toArguments(scope: Scope, tailBlockMode: Boolean): Arguments {
@@ -26,7 +30,7 @@ suspend fun Collection<ParsedArgument>.toArguments(scope: Scope, tailBlockMode:
     return Arguments(list,tailBlockMode)
 }
 
-data class Arguments(val list: List<Obj>,val tailBlockMode: Boolean = false) : List<Obj> by list {
+data class Arguments(val list: List<Obj>, val tailBlockMode: Boolean = false) : List<Obj> by list {
 
     constructor(vararg values: Obj) : this(values.toList())
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1,5 +1,6 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.*
 import net.sergeych.lyng.pacman.ImportProvider
 
 /**

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ListEntry.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Accessor
+
 sealed class ListEntry {
     data class Element(val accessor: Accessor) : ListEntry()
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/LoopBreakContinueException.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjVoid
+
 class LoopBreakContinueException(
     val doContinue: Boolean,
     val result: Obj = ObjVoid,

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ModuleScope.kt

@@ -1,5 +1,7 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.ObjRecord
+import net.sergeych.lyng.obj.ObjString
 import net.sergeych.lyng.pacman.ImportProvider
 
 /**

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ObjIterator.kt

@@ -1,3 +0,0 @@
-package net.sergeych.lyng
-
-val ObjIterator by lazy { ObjClass("Iterator") }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Scope.kt

@@ -1,5 +1,6 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.*
 import net.sergeych.lyng.pacman.ImportManager
 import net.sergeych.lyng.pacman.ImportProvider
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Script.kt

@@ -1,6 +1,7 @@
 package net.sergeych.lyng
 
 import kotlinx.coroutines.delay
+import net.sergeych.lyng.obj.*
 import net.sergeych.lyng.pacman.ImportManager
 import kotlin.math.*
 
@@ -114,7 +115,7 @@ class Script(
             }
             addFn( "abs" ) {
                 val x = args.firstAndOnly()
-                if( x is ObjInt ) ObjInt( x.value.absoluteValue ) else ObjReal( x.toDouble().absoluteValue )
+                if( x is ObjInt) ObjInt( x.value.absoluteValue ) else ObjReal( x.toDouble().absoluteValue )
             }
 
             addVoidFn("assert") {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/ScriptError.kt

@@ -2,6 +2,8 @@
 
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.ObjException
+
 open class ScriptError(val pos: Pos, val errorMessage: String, cause: Throwable? = null) : Exception(
     """
         $pos: Error: $errorMessage

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt

@@ -1,10 +1,12 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import net.sergeych.bintools.encodeToHex
+import net.sergeych.lyng.*
+import net.sergeych.lynon.LynonEncoder
 import net.sergeych.synctools.ProtectedOp
 import net.sergeych.synctools.withLock
 import kotlin.contracts.ExperimentalContracts
@@ -267,6 +269,9 @@ open class Obj {
     val asReadonly: ObjRecord by lazy { ObjRecord(this, false) }
     val asMutable: ObjRecord by lazy { ObjRecord(this, true) }
 
+    open suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        scope.raiseNotImplemented()
+    }
 
     companion object {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjArray.kt

@@ -1,4 +1,4 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 val ObjArray by lazy {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjArrayIterator.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjArrayIterator(val array: Obj) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBool.kt

@@ -1,4 +1,8 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
 
 data class ObjBool(val value: Boolean) : Obj() {
     override val asStr by lazy { ObjString(value.toString()) }
@@ -8,6 +12,10 @@ data class ObjBool(val value: Boolean) : Obj() {
         return value.compareTo(other.value)
     }
 
+    override fun hashCode(): Int {
+        return value.hashCode()
+    }
+
     override fun toString(): String = value.toString()
 
     override val objClass: ObjClass = type
@@ -22,8 +30,25 @@ data class ObjBool(val value: Boolean) : Obj() {
         return value
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        encoder.packBoolean(value)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjBool
+
+        return value == other.value
+    }
+
     companion object {
-        val type = ObjClass("Bool")
+        val type = object : ObjClass("Bool") {
+            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj {
+                return ObjBool(decoder.unpackBoolean())
+            }
+        }
     }
 }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt

@@ -1,7 +1,9 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
 import kotlin.math.min
 
 class ObjBuffer(val byteArray: UByteArray) : Obj() {
@@ -40,6 +42,10 @@ class ObjBuffer(val byteArray: UByteArray) : Obj() {
 
     val size by byteArray::size
 
+    override fun hashCode(): Int {
+        return byteArray.hashCode()
+    }
+
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
         if (other !is ObjBuffer) return super.compareTo(scope, other)
         val limit = min(size, other.size)
@@ -69,6 +75,15 @@ class ObjBuffer(val byteArray: UByteArray) : Obj() {
         return "Buffer(${byteArray.toList()})"
     }
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjBuffer
+
+        return byteArray contentEquals other.byteArray
+    }
+
     companion object {
         private suspend fun createBufferFrom(scope: Scope, obj: Obj): ObjBuffer =
             when (obj) {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjChar.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjChar(val value: Char): Obj() {
 
@@ -11,6 +13,19 @@ class ObjChar(val value: Char): Obj() {
 
     override fun inspect(): String = "'$value'"
 
+    override fun hashCode(): Int {
+        return value.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjChar
+
+        return value == other.value
+    }
+
     companion object {
         val type = ObjClass("Char").apply {
             addFn("code") { ObjInt(thisAs<ObjChar>().value.code.toLong()) }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjClass.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.*
+import net.sergeych.lynon.LynonDecoder
 
 val ObjClassType by lazy { ObjClass("Class") }
 
@@ -86,6 +89,8 @@ open class ObjClass(
         }
         return super.readField(scope, name)
     }
+
+    open fun deserialize(scope: Scope, decoder: LynonDecoder): Obj = scope.raiseNotImplemented()
 }
 
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjCollection.kt

@@ -1,4 +1,4 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 /**
  * Collection is an iterator with `size`]

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjDuration.kt

@@ -1,5 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
+import net.sergeych.lyng.Scope
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
 import kotlin.time.Duration.Companion.hours
@@ -21,6 +22,19 @@ class ObjDuration(val duration: Duration) : Obj() {
         else -1
     }
 
+    override fun hashCode(): Int {
+        return duration.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjDuration
+
+        return duration == other.duration
+    }
+
     companion object {
         val type = object : ObjClass("Duration") {
             override suspend fun callOn(scope: Scope): Obj {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstance.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Arguments
+import net.sergeych.lyng.Scope
 
 class ObjInstance(override val objClass: ObjClass) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstant.kt

@@ -1,9 +1,10 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.datetime.Clock
 import kotlinx.datetime.Instant
 import kotlinx.datetime.isDistantFuture
 import kotlinx.datetime.isDistantPast
+import net.sergeych.lyng.Scope
 
 class ObjInstant(val instant: Instant) : Obj() {
     override val objClass: ObjClass get() = type
@@ -38,6 +39,19 @@ class ObjInstant(val instant: Instant) : Obj() {
         return instant
     }
 
+    override fun hashCode(): Int {
+        return instant.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjInstant
+
+        return instant == other.instant
+    }
+
     companion object {
         val distantFuture by lazy {
             ObjInstant(Instant.DISTANT_FUTURE)

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt

@@ -1,4 +1,8 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
 
 class ObjInt(var value: Long,override val isConst: Boolean = false) : Obj(), Numeric {
     override val asStr get() = ObjString(value.toString())
@@ -93,10 +97,17 @@ class ObjInt(var value: Long,override val isConst: Boolean = false) : Obj(), Num
         return value == other.value
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        encoder.packSigned(value)
+    }
+
     companion object {
         val Zero = ObjInt(0, true)
         val One = ObjInt(1, true)
-        val type = ObjClass("Int")
+        val type = object: ObjClass("Int") {
+            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+                ObjInt(decoder.unpackSigned())
+        }
     }
 }
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjIterable.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Arguments
+import net.sergeych.lyng.Statement
 
 /**
  * Abstract class that must provide `iterator` method that returns [ObjIterator] instance.

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjIterator.kt

@@ -0,0 +1,3 @@
+package net.sergeych.lyng.obj
+
+val ObjIterator by lazy { ObjClass("Iterator") }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjKotlinIterator.kt

@@ -1,9 +1,10 @@
 @file:Suppress("unused")
 
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
+import net.sergeych.lyng.Scope
 
 /**
  * Iterator wrapper to allow Kotlin collections to be returned from Lyng objects;

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
 
 class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjMap.kt

@@ -1,4 +1,7 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Statement
 
 class ObjMapEntry(val key: Obj, val value: Obj) : Obj() {
 
@@ -55,6 +58,19 @@ class ObjMap(val map: MutableMap<Obj, Obj> = mutableMapOf()) : Obj() {
     }
     override fun toString(): String = map.toString()
 
+    override fun hashCode(): Int {
+        return map.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjMap
+
+        return map == other.map
+    }
+
     companion object {
 
         suspend fun listToMap(scope: Scope, list: List<Obj>): MutableMap<Obj, Obj> {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRange.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Obj() {
 
@@ -34,9 +36,9 @@ class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Ob
      * otherwise returns start.value.toInt()
      */
     fun startInt(scope: Scope): Int =
-        if( start == null || start is ObjNull ) 0
+        if( start == null || start is ObjNull) 0
         else {
-            if( start is ObjInt ) start.value.toInt()
+            if( start is ObjInt) start.value.toInt()
             else scope.raiseIllegalArgument("start is not Int: ${start.inspect()}")
         }
 
@@ -99,6 +101,27 @@ class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Ob
             ?: -1
     }
 
+    override fun hashCode(): Int {
+        var result = start?.hashCode() ?: 0
+        result = 31 * result + (end?.hashCode() ?: 0)
+        result = 31 * result + isEndInclusive.hashCode()
+        return result
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjRange
+
+        if (start != other.start) return false
+        if (end != other.end) return false
+        if (isEndInclusive != other.isEndInclusive) return false
+
+        return true
+    }
+
+
     companion object {
         val type = ObjClass("Range", ObjIterable).apply {
             addFn("start") {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjRangeIterator.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjRangeIterator(val self: ObjRange) : Obj() {
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjReal.kt

@@ -1,5 +1,10 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
+import net.sergeych.lyng.Pos
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
 import kotlin.math.floor
 import kotlin.math.roundToLong
 
@@ -56,8 +61,15 @@ data class ObjReal(val value: Double) : Obj(), Numeric {
         return value == other.value
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        encoder.packReal(value)
+    }
+
     companion object {
-        val type: ObjClass = ObjClass("Real").apply {
+        val type: ObjClass = object : ObjClass("Real") {
+            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+                ObjReal(decoder.unpackDouble())
+        }.apply {
             createField(
                 "roundToInt",
                 statement(Pos.builtIn) {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjSet.kt

@@ -1,4 +1,6 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
+
+import net.sergeych.lyng.Scope
 
 class ObjSet(val set: MutableSet<Obj> = mutableSetOf()) : Obj() {
 
@@ -65,6 +67,19 @@ class ObjSet(val set: MutableSet<Obj> = mutableSetOf()) : Obj() {
         }
     }
 
+    override fun hashCode(): Int {
+        return set.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ObjSet
+
+        return set == other.set
+    }
+
     companion object {
 
 

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt

@@ -1,7 +1,11 @@
-package net.sergeych.lyng
+package net.sergeych.lyng.obj
 
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.statement
+import net.sergeych.lynon.LynonDecoder
+import net.sergeych.lynon.LynonEncoder
 import net.sergeych.sprintf.sprintf
 
 @Serializable
@@ -36,8 +40,8 @@ data class ObjString(val value: String) : Obj() {
     }
 
     override suspend fun getAt(scope: Scope, index: Obj): Obj {
-        if( index is ObjInt ) return ObjChar(value[index.toInt()])
+        if( index is ObjInt) return ObjChar(value[index.toInt()])
-        if( index is ObjRange ) {
+        if( index is ObjRange) {
             val start = if(index.start == null || index.start.isNull) 0 else  index.start.toInt()
             val end = if( index.end  == null || index.end.isNull ) value.length else  {
                 val e = index.end.toInt()
@@ -73,8 +77,18 @@ data class ObjString(val value: String) : Obj() {
         return value == other.value
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        encoder.packBinaryData(value.encodeToByteArray())
+    }
+
     companion object {
-        val type = ObjClass("String").apply {
+        val type = object : ObjClass("String") {
+            override fun deserialize(scope: Scope, decoder: LynonDecoder): Obj =
+                ObjString(
+                    decoder.unpackBinaryData()?.decodeToString()
+                        ?: scope.raiseError("unexpected end of data")
+                )
+        }.apply {
             addFn("toInt") {
                 ObjInt(thisAs<ObjString>().value.toLong())
             }
@@ -119,7 +133,7 @@ data class ObjString(val value: String) : Obj() {
                 )
             }
             addFn("size") { ObjInt(thisAs<ObjString>().value.length.toLong()) }
-            addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble())}
+            addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble()) }
         }
     }
 }

lynglib/src/commonMain/kotlin/net/sergeych/lyng/statements.kt

@@ -1,5 +1,8 @@
 package net.sergeych.lyng
 
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjClass
+
 fun String.toSource(name: String = "eval"): Source = Source(name, this)
 
 sealed class ObjType {

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitInput.kt

@@ -55,7 +55,7 @@ abstract class BitInput {
         val tetrades = getBits(4).toInt()
         var result = 0UL
         var shift = 0
-        for (i in 0..<tetrades) {
+        for (i in 0.. tetrades) {
             result = result or (getBits(4) shl shift)
             shift += 4
         }
@@ -67,5 +67,18 @@ abstract class BitInput {
         val value = unpackUnsigned().toLong()
         return if( isNegative == 1) -value else value
     }
+
+    fun getBool(): Boolean {
+        return getBit() == 1
+    }
+
+    fun getBytes(count: Int): ByteArray? {
+        val result = ByteArray(count)
+        for (i in 0..<count) {
+            val b = getBitsOrNull(8) ?: return null
+            result[i] = b.toByte()
+        }
+        return result
+    }
 }
 

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitOutput.kt

@@ -36,7 +36,7 @@ abstract class BitOutput {
 
     fun packUnsigned(value: ULong) {
         val tetrades = sizeInTetrades(value)
-        putBits(tetrades, 4)
+        putBits(tetrades - 1, 4)
         var rest = value
         for( i in 0..<tetrades ) {
             putBits( rest and 0xFu, 4 )
@@ -67,4 +67,11 @@ abstract class BitOutput {
             isClosed = true
         }
     }
+
+    fun putBytes(data: ByteArray) {
+        for( b in data ) {
+            putBits(b.toULong(), 8)
+        }
+    }
+
 }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -0,0 +1,45 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjClass
+
+open class LynonDecoder(private val bin: BitInput) {
+
+    val cache = mutableListOf<Obj>()
+
+    fun unpackObject(scope: Scope, type: ObjClass): Obj {
+        return if( bin.getBit() == 0 ) {
+            // unpack and cache
+            val cached = bin.getBool()
+            type.deserialize(scope, this).also {
+                if( cached ) cache.add(it)
+            }
+        }
+        else {
+            // get cache reference
+            val size = sizeInBits(cache.size)
+            val id = bin.getBitsOrNull(size)?.toInt() ?: scope.raiseError("Invalid object id: unexpected end of stream")
+            if( id >= cache.size ) scope.raiseError("Invalid object id: $id should be in 0..<${cache.size}")
+            cache[id]
+        }
+    }
+
+    fun unpackBinaryData(): ByteArray? {
+        val size = bin.unpackUnsigned()
+        return bin.getBytes(size.toInt())
+    }
+
+    fun unpackBoolean(): Boolean {
+        return bin.getBit() == 1
+    }
+
+    fun unpackDouble(): Double {
+        return Double.fromBits(bin.getBits(64).toLong())
+    }
+
+    fun unpackSigned(): Long {
+        return bin.unpackSigned()
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -0,0 +1,62 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjBool
+import net.sergeych.lyng.obj.ObjChar
+import net.sergeych.lyng.obj.ObjInt
+
+class LynonPacker(private val bout: MemoryBitOutput= MemoryBitOutput()) : LynonEncoder(bout) {
+    fun toUByteArray(): UByteArray = bout.toUByteArray()
+}
+
+class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
+
+open class LynonEncoder(private val bout: BitOutput) {
+
+    fun shouldCache(obj: Obj): Boolean = when (obj) {
+        is ObjChar -> false
+        is ObjInt -> obj.value > 0x10000FF
+        is ObjBool -> false
+        else -> true
+    }
+
+    val cache = mutableMapOf<Obj,Int>()
+
+    suspend fun packObject(scope: Scope,obj: Obj) {
+        cache[obj]?.let { cacheId ->
+            val size = sizeInBits(cache.size)
+            bout.putBit(1)
+            bout.putBits(cacheId, size)
+        } ?: run {
+            bout.putBit(0)
+            if( shouldCache(obj) ) {
+                bout.putBit(1)
+                cache[obj] = cache.size
+            }
+            else
+                bout.putBit(0)
+            obj.serialize(scope, this)
+        }
+    }
+
+    fun packBinaryData(data: ByteArray) {
+        bout.packUnsigned(data.size.toULong())
+        bout.putBytes(data)
+    }
+
+    fun packSigned(value: Long) { bout.packSigned(value) }
+    @Suppress("unused")
+    fun packUnsigned(value: ULong) { bout.packUnsigned(value) }
+
+    @Suppress("unused")
+    fun packBool(value: Boolean) { bout.putBit(if (value) 1 else 0) }
+    fun packReal(value: Double) {
+        bout.putBits(value.toRawBits().toULong(), 64)
+    }
+
+    fun packBoolean(value: Boolean) {
+        bout.putBit(if (value) 1 else 0)
+    }
+
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/MemoryBitInput.kt

@@ -1,6 +1,8 @@
 package net.sergeych.lynon
 
 class MemoryBitInput(val packedBits: UByteArray): BitInput() {
+    constructor(bout: MemoryBitOutput): this(bout.toUByteArray())
+
     private var index = 0
 
     override fun getByte(): Int {

lynglib/src/commonMain/kotlin/net/sergeych/lynon/bit_tools.kt

@@ -30,4 +30,6 @@ fun sizeInBits(value: ULong): Int {
         rest = rest shr 1
     }
     return size
-}
+}
+
+fun sizeInBits(value: Int): Int = sizeInBits(value.toULong())

lynglib/src/commonTest/kotlin/ScriptTest.kt

@@ -3,6 +3,7 @@ import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.test.runTest
 import net.sergeych.lyng.*
+import net.sergeych.lyng.obj.*
 import net.sergeych.lyng.pacman.InlineSourcesImportProvider
 import kotlin.test.*
 

lynglib/src/jvmTest/kotlin/BookTest.kt

@@ -4,8 +4,8 @@ import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.flowOn
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.test.runTest
-import net.sergeych.lyng.ObjVoid
 import net.sergeych.lyng.Scope
+import net.sergeych.lyng.obj.ObjVoid
 import java.nio.file.Files
 import java.nio.file.Files.readAllLines
 import java.nio.file.Paths

lynglib/src/jvmTest/kotlin/LynonTests.kt

@@ -1,8 +1,8 @@
-import junit.framework.TestCase.assertEquals
+import junit.framework.TestCase.*
-import net.sergeych.bintools.toDump
+import kotlinx.coroutines.test.runTest
-import net.sergeych.lynon.MemoryBitInput
+import net.sergeych.lyng.Scope
-import net.sergeych.lynon.MemoryBitOutput
+import net.sergeych.lyng.obj.*
-import net.sergeych.lynon.sizeInTetrades
+import net.sergeych.lynon.*
 import kotlin.test.Test
 
 class LynonTests {
@@ -19,14 +19,23 @@ class LynonTests {
         assertEquals(3, sizeInTetrades(257u))
     }
 
+    @Test
+    fun testSizeInBits() {
+        assertEquals(1, sizeInBits(0u))
+        assertEquals(1, sizeInBits(1u))
+        assertEquals(2, sizeInBits(2u))
+        assertEquals(2, sizeInBits(3u))
+        assertEquals(4, sizeInBits(15u))
+    }
+
     @Test
     fun testBitStreams() {
 
         val bout = MemoryBitOutput()
         bout.putBits(2, 3)
         bout.putBits(1, 7)
-        bout.putBits( 197, 8)
+        bout.putBits(197, 8)
-        bout.putBits( 3, 4)
+        bout.putBits(3, 4)
         bout.close()
 
         val bin = MemoryBitInput(bout.toUByteArray())
@@ -41,11 +50,28 @@ class LynonTests {
         val bout = MemoryBitOutput()
         bout.packUnsigned(1471792UL)
         bout.close()
-        println(bout.toUByteArray().toDump())
         val bin = MemoryBitInput(bout.toUByteArray())
         assertEquals(1471792UL, bin.unpackUnsigned())
     }
 
+    @Test
+    fun testUnsignedPackLongInteger() {
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(ULong.MAX_VALUE)
+        bout.close()
+        val bin = MemoryBitInput(bout.toUByteArray())
+        assertEquals(ULong.MAX_VALUE, bin.unpackUnsigned())
+    }
+
+    @Test
+    fun testUnsignedPackLongSmallInteger() {
+        val bout = MemoryBitOutput()
+        bout.packUnsigned(7UL)
+        bout.close()
+        val bin = MemoryBitInput(bout.toUByteArray())
+        assertEquals(7UL, bin.unpackUnsigned())
+    }
+
     @Test
     fun testSignedPackInteger() {
         val bout = MemoryBitOutput()
@@ -53,10 +79,114 @@ class LynonTests {
         bout.packSigned(1471792L)
 //        bout.packSigned(147179L)
         bout.close()
-        println(bout.toUByteArray().toDump())
         val bin = MemoryBitInput(bout.toUByteArray())
         assertEquals(-1471792L, bin.unpackSigned())
         assertEquals(1471792L, bin.unpackSigned())
     }
 
+    @Test
+    fun testCache1() = runTest {
+        val bout = MemoryBitOutput()
+        val encoder = LynonEncoder(bout)
+        val s = "Hello, World!".toObj()
+        val scope = Scope()
+        encoder.packObject(scope, s) // 1
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s)
+        encoder.packObject(scope, s) // 8
+
+        val decoder = LynonDecoder(MemoryBitInput(bout))
+        val s1 = decoder.unpackObject(scope, ObjString.type) // 1
+        assertEquals(s, s1)
+        assertNotSame(s, s1)
+        val s2 = decoder.unpackObject(scope, ObjString.type)
+        assertEquals(s, s2)
+        assertSame(s1, s2)
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type))
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type))
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type))
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type))
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type))
+        assertSame(s1, decoder.unpackObject(scope, ObjString.type)) // 8
+    }
+
+    @Test
+    fun testCache2() = runTest {
+        val variants = (100..500).map { "Sample $it".toObj() }.shuffled()
+        var source = variants.shuffled()
+        for (i in 0..300) source += variants.shuffled()
+        val encoder = LynonPacker()
+        val scope = Scope()
+        for (s in source) {
+            encoder.packObject(scope, s)
+        }
+        val decoder = LynonUnpacker(encoder.toUByteArray())
+        val restored = mutableListOf<Obj>()
+        for (i in source.indices) {
+            restored.add(decoder.unpackObject(scope, ObjString.type))
+        }
+        assertEquals(restored, source)
+    }
+
+    @Test
+    fun testUnpackBoolean() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            packObject(scope, ObjBool(true))
+            packObject(scope, ObjBool(false))
+            packObject(scope, ObjBool(true))
+            packObject(scope, ObjBool(true))
+        }.toUByteArray())
+        assertEquals(ObjTrue, decoder.unpackObject(scope, ObjBool.type))
+        assertEquals(ObjFalse, decoder.unpackObject(scope, ObjBool.type))
+        assertEquals(ObjTrue, decoder.unpackObject(scope, ObjBool.type))
+        assertEquals(ObjTrue, decoder.unpackObject(scope, ObjBool.type))
+    }
+
+    @Test
+    fun testUnpackReal() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            packObject(scope, ObjReal(-Math.PI))
+            packObject(scope, ObjReal(Math.PI))
+            packObject(scope, ObjReal(-Math.PI))
+            packObject(scope, ObjReal(Math.PI))
+            packObject(scope, ObjReal(Double.NaN))
+            packObject(scope, ObjReal(Double.NEGATIVE_INFINITY))
+            packObject(scope, ObjReal(Double.POSITIVE_INFINITY))
+            packObject(scope, ObjReal(Double.MIN_VALUE))
+            packObject(scope, ObjReal(Double.MAX_VALUE))
+        }.toUByteArray())
+        assertEquals(ObjReal(-Math.PI), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Math.PI), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(-Math.PI), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Math.PI), decoder.unpackObject(scope, ObjReal.type))
+        assert((decoder.unpackObject(scope, ObjReal.type)).toDouble().isNaN())
+        assertEquals(ObjReal(Double.NEGATIVE_INFINITY), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.POSITIVE_INFINITY), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.MIN_VALUE), decoder.unpackObject(scope, ObjReal.type))
+        assertEquals(ObjReal(Double.MAX_VALUE), decoder.unpackObject(scope, ObjReal.type))
+    }
+    @Test
+    fun testUnpackInt() = runTest {
+        val scope = Scope()
+        val decoder = LynonUnpacker(LynonPacker().apply {
+            packObject(scope, ObjInt(0))
+            packObject(scope, ObjInt(-1))
+            packObject(scope, ObjInt(23))
+            packObject(scope, ObjInt(Long.MIN_VALUE))
+            packObject(scope, ObjInt(Long.MAX_VALUE))
+            packObject(scope, ObjInt(Long.MAX_VALUE))
+        }.toUByteArray())
+        assertEquals(ObjInt(0), decoder.unpackObject(scope, ObjInt.type))
+        assertEquals(ObjInt(-1), decoder.unpackObject(scope, ObjInt.type))
+        assertEquals(ObjInt(23), decoder.unpackObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MIN_VALUE), decoder.unpackObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
+        assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
+    }
 }