refs #35 started user class serialization; started MP compression

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1112,6 +1112,7 @@ class Compiler(
         // inheritance must alter this code:
         val newClass = ObjClass(className).apply {
             instanceConstructor = constructorCode
+            constructorMeta = constructorArgsDeclaration
         }
 
         return statement {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBool.kt

@@ -31,7 +31,7 @@ data class ObjBool(val value: Boolean) : Obj() {
     }
 
     override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
-        encoder.packBoolean(value)
+        encoder.encodeBoolean(value)
     }
 
     override fun equals(other: Any?): Boolean {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjClass.kt

@@ -10,6 +10,7 @@ open class ObjClass(
     vararg parents: ObjClass,
 ) : Obj() {
 
+    var constructorMeta: ArgsDeclaration? = null
     var instanceConstructor: Statement? = null
 
     val allParentsSet: Set<ObjClass> =

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstance.kt

@@ -2,6 +2,7 @@ package net.sergeych.lyng.obj
 
 import net.sergeych.lyng.Arguments
 import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonEncoder
 
 class ObjInstance(override val objClass: ObjClass) : Obj() {
 
@@ -44,6 +45,18 @@ class ObjInstance(override val objClass: ObjClass) : Obj() {
         return "${objClass.className}($fields)"
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        val meta = objClass.constructorMeta
+            ?: scope.raiseError("can't serialize non-serializable object (no constructor meta)")
+        for( p in meta.params) {
+            val r = readField(scope, p.name)
+            println("serialize ${p.name}=${r.value}")
+            TODO()
+//            encoder.encodeObj(scope, r.value)
+        }
+        // todo: possible vars?
+    }
+
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
         if (other !is ObjInstance) return -1
         if (other.objClass != objClass) return -1

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt

@@ -98,7 +98,7 @@ class ObjInt(var value: Long,override val isConst: Boolean = false) : Obj(), Num
     }
 
     override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
-        encoder.packSigned(value)
+        encoder.encodeSigned(value)
     }
 
     companion object {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjReal.kt

@@ -62,7 +62,7 @@ data class ObjReal(val value: Double) : Obj(), Numeric {
     }
 
     override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
-        encoder.packReal(value)
+        encoder.encodeReal(value)
     }
 
     companion object {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt

@@ -78,7 +78,7 @@ data class ObjString(val value: String) : Obj() {
     }
 
     override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
-        encoder.packBinaryData(value.encodeToByteArray())
+        encoder.encodeBinaryData(value.encodeToByteArray())
     }
 
     companion object {

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -4,27 +4,30 @@ import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.Obj
 import net.sergeych.lyng.obj.ObjClass
 
-open class LynonDecoder(private val bin: BitInput) {
+open class LynonDecoder(val bin: BitInput,val settings: LynonSettings = LynonSettings.default) {
 
     val cache = mutableListOf<Obj>()
 
-    fun unpackObject(scope: Scope, type: ObjClass): Obj {
+    inline fun decodeCached(f: LynonDecoder.() -> Obj): Obj {
         return if( bin.getBit() == 0 ) {
             // unpack and cache
-            val cached = bin.getBool()
+            f().also {
-            type.deserialize(scope, this).also {
+                if( settings.shouldCache(it) ) cache.add(it)
-                if( cached ) cache.add(it)
             }
         }
         else {
             // get cache reference
             val size = sizeInBits(cache.size)
-            val id = bin.getBitsOrNull(size)?.toInt() ?: scope.raiseError("Invalid object id: unexpected end of stream")
+            val id = bin.getBitsOrNull(size)?.toInt() ?: throw RuntimeException("Invalid object id: unexpected end of stream")
-            if( id >= cache.size ) scope.raiseError("Invalid object id: $id should be in 0..<${cache.size}")
+            if( id >= cache.size ) throw RuntimeException("Invalid object id: $id should be in 0..<${cache.size}")
             cache[id]
         }
     }
 
+    fun unpackObject(scope: Scope, type: ObjClass): Obj {
+        return decodeCached { type.deserialize(scope, this) }
+    }
+
     fun unpackBinaryData(): ByteArray? {
         val size = bin.unpackUnsigned()
         return bin.getBytes(size.toInt())

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -2,60 +2,56 @@ package net.sergeych.lynon
 
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.Obj
-import net.sergeych.lyng.obj.ObjBool
-import net.sergeych.lyng.obj.ObjChar
-import net.sergeych.lyng.obj.ObjInt
 
-class LynonPacker(private val bout: MemoryBitOutput= MemoryBitOutput()) : LynonEncoder(bout) {
+open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonSettings.default) {
-    fun toUByteArray(): UByteArray = bout.toUByteArray()
-}
 
-class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
+    val cache = mutableMapOf<Any, Int>()
 
-open class LynonEncoder(private val bout: BitOutput) {
+    private inline fun encodeCached(item: Any, packer: LynonEncoder.() -> Unit) {
-
+        if (item is Obj) {
-    fun shouldCache(obj: Obj): Boolean = when (obj) {
+            cache[item]?.let { cacheId ->
-        is ObjChar -> false
-        is ObjInt -> obj.value > 0x10000FF
-        is ObjBool -> false
-        else -> true
-    }
-
-    val cache = mutableMapOf<Obj,Int>()
-
-    suspend fun packObject(scope: Scope,obj: Obj) {
-        cache[obj]?.let { cacheId ->
                 val size = sizeInBits(cache.size)
                 bout.putBit(1)
-            bout.putBits(cacheId, size)
+                bout.putBits(cacheId.toULong(), size)
             } ?: run {
                 bout.putBit(0)
-            if( shouldCache(obj) ) {
+                if (settings.shouldCache(item))
-                bout.putBit(1)
+                    cache[item] = cache.size
-                cache[obj] = cache.size
+                packer()
             }
-            else
+        }
-                bout.putBit(0)
+    }
+
+    suspend fun encodeObj(scope: Scope, obj: Obj) {
+        encodeCached(obj) {
             obj.serialize(scope, this)
         }
     }
 
-    fun packBinaryData(data: ByteArray) {
+    fun encodeBinaryData(data: ByteArray) {
         bout.packUnsigned(data.size.toULong())
         bout.putBytes(data)
     }
 
-    fun packSigned(value: Long) { bout.packSigned(value) }
+    fun encodeSigned(value: Long) {
-    @Suppress("unused")
+        bout.packSigned(value)
-    fun packUnsigned(value: ULong) { bout.packUnsigned(value) }
+    }
 
     @Suppress("unused")
-    fun packBool(value: Boolean) { bout.putBit(if (value) 1 else 0) }
+    fun encodeUnsigned(value: ULong) {
-    fun packReal(value: Double) {
+        bout.packUnsigned(value)
+    }
+
+    @Suppress("unused")
+    fun encodeBool(value: Boolean) {
+        bout.putBit(if (value) 1 else 0)
+    }
+
+    fun encodeReal(value: Double) {
         bout.putBits(value.toRawBits().toULong(), 64)
     }
 
-    fun packBoolean(value: Boolean) {
+    fun encodeBoolean(value: Boolean) {
         bout.putBit(if (value) 1 else 0)
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonSettings.kt

@@ -0,0 +1,20 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjBool
+import net.sergeych.lyng.obj.ObjChar
+import net.sergeych.lyng.obj.ObjInt
+
+open class LynonSettings() {
+
+    open fun shouldCache(obj: Obj): Boolean = when (obj) {
+        is ObjChar -> false
+        is ObjInt -> obj.value > 0x10000FF
+        is ObjBool -> false
+        else -> true
+    }
+
+    companion object {
+        val default = LynonSettings()
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw0.kt

@@ -0,0 +1,96 @@
+package net.sergeych.lynon
+
+import net.sergeych.bintools.ByteChunk
+import kotlin.math.roundToInt
+
+/**
+ * LZW compression algorithm: work in progress.
+ *
+ * Uses Lyng but input/output. Uses automatic code size.
+ *
+ * TODO: - reset dictionary
+ */
+class LZW {
+    companion object {
+
+        val MAX_CODE_SIZE = 12
+        val STOP_CODE = (1 shl MAX_CODE_SIZE) - 1
+        val MAX_DICT_SIZE = (STOP_CODE * 0.92).roundToInt()
+
+
+        /**
+         * Compresses the input string using LZW algorithm
+         * @param input The string to compress
+         * @return List of compressed codes
+         */
+        fun compress(input: UByteArray,bitOutput: BitOutput) {
+            // Initialize dictionary with all possible single characters
+            val dictionary = mutableMapOf<ByteChunk, Int>()
+            for (i in 0..255) {
+                // 23
+                dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
+            }
+
+            var nextCode = 256
+            var current = ByteChunk(ubyteArrayOf())
+//            val result = mutableListOf<Int>()
+
+            for (char in input) {
+                val combined = current + char
+                if (dictionary.containsKey(combined)) {
+                    current = combined
+                } else {
+                    val size = sizeInBits(dictionary.size)
+                    bitOutput.putBits(dictionary[current]!!,size)
+                    dictionary[combined] = nextCode++
+                    current = ByteChunk(ubyteArrayOf(char))
+                }
+            }
+
+            if (current.size > 0) {
+                val size = sizeInBits(dictionary.size)
+                bitOutput.putBits(dictionary[current]!!,size)
+            }
+        }
+
+        /**
+         * Decompresses a list of LZW codes back to the original string
+         * @param compressed The list of compressed codes
+         * @return The decompressed string
+         */
+        fun decompress(compressed: BitInput): UByteArray {
+            // Initialize dictionary with all possible single characters
+            val dictionary = mutableMapOf<Int, UByteArray>()
+            for (i in 0..255) {
+                dictionary[i] = ubyteArrayOf(i.toUByte())
+            }
+
+            var nextCode = 256
+            var previous = dictionary[compressed.getBits(9).toInt()]!!
+            val result = mutableListOf<UByte>()
+
+            while( !compressed.isEndOfStream ) {
+                val codeSize = sizeInBits(nextCode + 1)
+                val code = compressed.getBitsOrNull(codeSize)?.toInt() ?: break
+                val current = if ( code in dictionary) {
+                    dictionary[code]!!
+                } else if (code == nextCode) {
+                    // Special case for pattern like cScSc
+                    previous + previous[0]
+                } else {
+                    throw IllegalArgumentException("Invalid compressed code: $code")
+                }
+
+                result += current
+                dictionary[nextCode++] = previous + current[0]
+                previous = current
+            }
+
+            return result.toTypedArray().toUByteArray()
+        }
+    }
+}
+
+private operator fun ByteChunk.plus(byte: UByte): ByteChunk {
+    return ByteChunk(data + byte)
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt

@@ -0,0 +1,14 @@
+package net.sergeych.lynon
+
+/**
+ * Variant of [LynonEncoder] that writes to embedded [MemoryBitOutput]
+ */
+class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput(), settings: LynonSettings = LynonSettings.default)
+    : LynonEncoder(bout, settings) {
+    fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
+}
+
+/**
+ * Variant of [LynonDecoder] that reads from a given `source` using [MemoryBitInput]
+ */
+class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))

lynglib/src/jvmTest/kotlin/LynonTests.kt

@@ -3,6 +3,8 @@ import kotlinx.coroutines.test.runTest
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.*
 import net.sergeych.lynon.*
+import java.nio.file.Files
+import java.nio.file.Path
 import kotlin.test.Test
 
 class LynonTests {
@@ -90,14 +92,14 @@ class LynonTests {
         val encoder = LynonEncoder(bout)
         val s = "Hello, World!".toObj()
         val scope = Scope()
-        encoder.packObject(scope, s) // 1
+        encoder.encodeObj(scope, s) // 1
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s)
+        encoder.encodeObj(scope, s)
-        encoder.packObject(scope, s) // 8
+        encoder.encodeObj(scope, s) // 8
 
         val decoder = LynonDecoder(MemoryBitInput(bout))
         val s1 = decoder.unpackObject(scope, ObjString.type) // 1
@@ -122,7 +124,7 @@ class LynonTests {
         val encoder = LynonPacker()
         val scope = Scope()
         for (s in source) {
-            encoder.packObject(scope, s)
+            encoder.encodeObj(scope, s)
         }
         val decoder = LynonUnpacker(encoder.toUByteArray())
         val restored = mutableListOf<Obj>()
@@ -136,10 +138,10 @@ class LynonTests {
     fun testUnpackBoolean() = runTest {
         val scope = Scope()
         val decoder = LynonUnpacker(LynonPacker().apply {
-            packObject(scope, ObjBool(true))
+            encodeObj(scope, ObjBool(true))
-            packObject(scope, ObjBool(false))
+            encodeObj(scope, ObjBool(false))
-            packObject(scope, ObjBool(true))
+            encodeObj(scope, ObjBool(true))
-            packObject(scope, ObjBool(true))
+            encodeObj(scope, ObjBool(true))
         }.toUByteArray())
         assertEquals(ObjTrue, decoder.unpackObject(scope, ObjBool.type))
         assertEquals(ObjFalse, decoder.unpackObject(scope, ObjBool.type))
@@ -151,15 +153,15 @@ class LynonTests {
     fun testUnpackReal() = runTest {
         val scope = Scope()
         val decoder = LynonUnpacker(LynonPacker().apply {
-            packObject(scope, ObjReal(-Math.PI))
+            encodeObj(scope, ObjReal(-Math.PI))
-            packObject(scope, ObjReal(Math.PI))
+            encodeObj(scope, ObjReal(Math.PI))
-            packObject(scope, ObjReal(-Math.PI))
+            encodeObj(scope, ObjReal(-Math.PI))
-            packObject(scope, ObjReal(Math.PI))
+            encodeObj(scope, ObjReal(Math.PI))
-            packObject(scope, ObjReal(Double.NaN))
+            encodeObj(scope, ObjReal(Double.NaN))
-            packObject(scope, ObjReal(Double.NEGATIVE_INFINITY))
+            encodeObj(scope, ObjReal(Double.NEGATIVE_INFINITY))
-            packObject(scope, ObjReal(Double.POSITIVE_INFINITY))
+            encodeObj(scope, ObjReal(Double.POSITIVE_INFINITY))
-            packObject(scope, ObjReal(Double.MIN_VALUE))
+            encodeObj(scope, ObjReal(Double.MIN_VALUE))
-            packObject(scope, ObjReal(Double.MAX_VALUE))
+            encodeObj(scope, ObjReal(Double.MAX_VALUE))
         }.toUByteArray())
         assertEquals(ObjReal(-Math.PI), decoder.unpackObject(scope, ObjReal.type))
         assertEquals(ObjReal(Math.PI), decoder.unpackObject(scope, ObjReal.type))
@@ -175,12 +177,12 @@ class LynonTests {
     fun testUnpackInt() = runTest {
         val scope = Scope()
         val decoder = LynonUnpacker(LynonPacker().apply {
-            packObject(scope, ObjInt(0))
+            encodeObj(scope, ObjInt(0))
-            packObject(scope, ObjInt(-1))
+            encodeObj(scope, ObjInt(-1))
-            packObject(scope, ObjInt(23))
+            encodeObj(scope, ObjInt(23))
-            packObject(scope, ObjInt(Long.MIN_VALUE))
+            encodeObj(scope, ObjInt(Long.MIN_VALUE))
-            packObject(scope, ObjInt(Long.MAX_VALUE))
+            encodeObj(scope, ObjInt(Long.MAX_VALUE))
-            packObject(scope, ObjInt(Long.MAX_VALUE))
+            encodeObj(scope, ObjInt(Long.MAX_VALUE))
         }.toUByteArray())
         assertEquals(ObjInt(0), decoder.unpackObject(scope, ObjInt.type))
         assertEquals(ObjInt(-1), decoder.unpackObject(scope, ObjInt.type))
@@ -189,4 +191,27 @@ class LynonTests {
         assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
         assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
     }
+
+    @Test
+    fun testLzw() {
+        // Example usage
+//        val original = "TOBEORNOTTOBEORTOBEORNOT"
+        val original = Files.readString(Path.of("../sample_texts/dikkens_hard_times.txt"))
+//        println("Original: $original")
+        println("Length: ${original.length}")
+
+        // Compress
+        val out = MemoryBitOutput()
+        LZW.compress(original.encodeToByteArray().toUByteArray(), out)
+//        println("\nCompressed codes: ${out.toUByteArray().toDump()}")
+        println("Number of codes: ${out.toUByteArray().size}")
+
+//        // Decompress
+        val decompressed = LZW.decompress(MemoryBitInput(out.toUByteArray())).toByteArray().decodeToString()
+//        println("\nDecompressed: $decompressed")
+        println("Length: ${decompressed.length}")
+
+        // Verification
+        println("\nOriginal and decompressed match: ${original == decompressed}")
+    }
 }