list serialization (optimized, homogenous and heterogeneous

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -17,10 +17,11 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe
     val cache = mutableListOf<Any>()
 
 
-    inline fun <T : Any>decodeCached(f: LynonDecoder.() -> T): T {
+    inline fun <reified T : Any>decodeCached(f: LynonDecoder.() -> T): T {
         return if (bin.getBit() == 0) {
             // unpack and cache
             f().also {
+//                println("decode: cache miss: ${cache.size}: $it:${it::class.simpleName}")
                 if (settings.shouldCache(it)) cache.add(it)
             }
         } else {
@@ -29,7 +30,8 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe
             val id = bin.getBitsOrNull(size)?.toInt()
                 ?: throw RuntimeException("Invalid object id: unexpected end of stream")
             if (id >= cache.size) throw RuntimeException("Invalid object id: $id should be in 0..<${cache.size}")
-            @Suppress("UNCHECKED_CAST")
+//            println("decode: cache hit ${id}: ${cache[id]}:${cache[id]::class.simpleName}")
+//            @Suppress("UNCHECKED_CAST")
             cache[id] as T
         }
     }
@@ -39,8 +41,29 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe
         type.objClass.deserialize(scope, this, type)
     }
 
-    suspend fun decodeObject(scope: Scope, type: ObjClass): Obj {
-        return decodeCached {  type.deserialize(scope, this, null) }
+    suspend fun decodeAnyList(scope: Scope): MutableList<Obj> {
+        return if( bin.getBit() == 1) {
+            // homogenous
+            val type = LynonType.entries[getBitsAsInt(4)]
+            val size = bin.unpackUnsigned().toInt()
+            println("detected homogenous list type $type, $size items")
+            val list = mutableListOf<Obj>()
+            val objClass = type.objClass
+            for( i in 0 ..< size) {
+                list += decodeObject(scope, objClass, type).also {
+                        println("decoded: $it")
+                }
+            }
+            list
+        }
+        else {
+            val size = unpackUnsigned().toInt()
+            (0..<size).map { decodeAny(scope) }.toMutableList()
+        }
+    }
+
+    suspend fun decodeObject(scope: Scope, type: ObjClass,overrideType: LynonType?=null): Obj {
+        return decodeCached {  type.deserialize(scope, this, overrideType) }
     }
 
     fun unpackBinaryData(): ByteArray = bin.decompress()

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -4,21 +4,45 @@ import net.sergeych.bintools.ByteChunk
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.*
 
-enum class LynonType(val objClass: ObjClass,val defaultFrequency: Int = 1) {
+enum class LynonType(val objClass: ObjClass, val defaultFrequency: Int = 1) {
     Null(ObjNull.objClass, 80),
     Int0(ObjInt.type, 70),
     IntNegative(ObjInt.type, 50),
     IntPositive(ObjInt.type, 100),
+    IntSigned(ObjInt.type, 30),
     String(ObjString.type, 100),
     Real(ObjReal.type),
     Bool(ObjBool.type, 80),
     List(ObjList.type, 70),
-    Map(ObjMap.type,40),
+    Map(ObjMap.type, 40),
     Set(ObjSet.type),
     Buffer(ObjBuffer.type, 50),
     Instant(ObjInstant.type, 30),
     Duration(ObjDuration.type),
-    Other(Obj.rootObjectType,60);
+    Other(Obj.rootObjectType, 60);
+
+    fun generalizeTo(other: LynonType): LynonType? {
+        if (this == other) return this
+        return (if (this.isInt && other.isInt) {
+            when {
+                this == Int0 -> other // upgrade 0 to some other int
+                other == Int0 -> this // 0 is member of our class, ignore
+                // different signum propagate to signed
+                else -> IntSigned
+            }
+        } else
+        // impossible to generalize
+            null
+                ).also { println("Gen $this + $other -> $it") }
+    }
+
+    val isInt by lazy {
+        when (this) {
+            Int0, IntSigned, IntPositive, IntNegative -> true
+            else -> false
+        }
+    }
+
 }
 
 open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = LynonSettings.default) {
@@ -29,14 +53,20 @@ open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = Lynon
 
         suspend fun serializeAndCache(key: Any = item) {
             cache[key]?.let { cacheId ->
+//                println("encode: Cache hit: ${cacheId}: $item: ${item::class.simpleName}")
                 val size = sizeInBits(cache.size)
                 bout.putBit(1)
                 bout.putBits(cacheId.toULong(), size)
             } ?: run {
                 bout.putBit(0)
-                if (settings.shouldCache(item))
+                if (settings.shouldCache(item)) {
+//                    println("encode add cache: ${cache.size}: $item: ${item::class.simpleName}")
+                    packer()
                     cache[key] = cache.size
-                packer()
+                } else {
+//                    println("encode but not cache $item")
+                    packer()
+                }
             }
         }
 
@@ -53,21 +83,66 @@ open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = Lynon
      *
      * Caching is used automatically.
      */
-    suspend fun encodeAny(scope: Scope, value: Obj) {
-        encodeCached(value) {
-            val type = value.lynonType()
-            putType(type)
-            value.serialize(scope, this, type)
+    suspend fun encodeAny(scope: Scope, obj: Obj) {
+        encodeCached(obj) {
+            val type = putTypeRecord(obj, obj.lynonType())
+            obj.serialize(scope, this, type)
         }
     }
 
+    private fun putTypeRecord(obj: Obj, type: LynonType): LynonType {
+        putType(type)
+        return type
+    }
+
     private fun putType(type: LynonType) {
         bout.putBits(type.ordinal.toULong(), 4)
     }
 
-    suspend fun encodeObject(scope: Scope, obj: Obj) {
+    /**
+     * AnyList could be homogenous (first bit=1) and heterogeneous (first bit=0). Homogenous list
+     * has a single type record that precedes the list, heterogeneous hash typed record
+     * for each item.
+     *
+     */
+    suspend fun encodeAnyList(scope: Scope, list: List<Obj>) {
+        val objClass = list[0].objClass
+        var type = list[0].lynonType()
+        var isHomogeneous = true
+        for (i in list.drop(1))
+            if (i.objClass != objClass) {
+                isHomogeneous = false
+                break
+            } else {
+                // same class but type might need generalization
+                type = type.generalizeTo(i.lynonType())
+                    ?: scope.raiseError("inner error: can't generalize lynon type $type to ${i.lynonType()}")
+            }
+        if (isHomogeneous) {
+            putBit(1)
+            putTypeRecord(list[0], type)
+            encodeUnsigned(list.size.toULong())
+            for (i in list) encodeObject(scope, i, type)
+        } else {
+            putBit(0)
+            encodeUnsigned(list.size.toULong())
+            for (i in list) encodeAny(scope, i)
+        }
+
+    }
+
+    /**
+     * Write object _with no type record_: type is known
+     */
+    suspend fun encodeObject(scope: Scope, obj: Obj,overrideType: LynonType? = null) {
         encodeCached(obj) {
-            obj.serialize(scope, this, null)
+            obj.serialize(scope, this, overrideType)
+        }
+    }
+
+    suspend fun encodeCachedBytes(bytes: ByteArray) {
+        encodeCached(bytes) {
+            bout.compress(bytes)
         }
     }