refs #35 started user class serialization; started MP compression

lynglib/src/commonMain/kotlin/net/sergeych/lyng/Compiler.kt

@@ -1112,6 +1112,7 @@ class Compiler(
         // inheritance must alter this code:
         val newClass = ObjClass(className).apply {
             instanceConstructor = constructorCode
+            constructorMeta = constructorArgsDeclaration
         }
 
         return statement {

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjClass.kt

@@ -10,6 +10,7 @@ open class ObjClass(
     vararg parents: ObjClass,
 ) : Obj() {
 
+    var constructorMeta: ArgsDeclaration? = null
     var instanceConstructor: Statement? = null
 
     val allParentsSet: Set<ObjClass> =

lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInstance.kt

@@ -2,6 +2,7 @@ package net.sergeych.lyng.obj
 
 import net.sergeych.lyng.Arguments
 import net.sergeych.lyng.Scope
+import net.sergeych.lynon.LynonEncoder
 
 class ObjInstance(override val objClass: ObjClass) : Obj() {
 
@@ -44,6 +45,18 @@ class ObjInstance(override val objClass: ObjClass) : Obj() {
         return "${objClass.className}($fields)"
     }
 
+    override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
+        val meta = objClass.constructorMeta
+            ?: scope.raiseError("can't serialize non-serializable object (no constructor meta)")
+        for( p in meta.params) {
+            val r = readField(scope, p.name)
+            println("serialize ${p.name}=${r.value}")
+            TODO()
+//            encoder.encodeObj(scope, r.value)
+        }
+        // todo: possible vars?
+    }
+
     override suspend fun compareTo(scope: Scope, other: Obj): Int {
         if (other !is ObjInstance) return -1
         if (other.objClass != objClass) return -1

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt

@@ -4,16 +4,15 @@ import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.Obj
 import net.sergeych.lyng.obj.ObjClass
 
-open class LynonDecoder(val bin: BitInput) {
+open class LynonDecoder(val bin: BitInput,val settings: LynonSettings = LynonSettings.default) {
 
     val cache = mutableListOf<Obj>()
 
     inline fun decodeCached(f: LynonDecoder.() -> Obj): Obj {
         return if( bin.getBit() == 0 ) {
             // unpack and cache
-            val cached = bin.getBool()
             f().also {
-                if( cached ) cache.add(it)
+                if( settings.shouldCache(it) ) cache.add(it)
             }
         }
         else {

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonEncoder.kt

@@ -2,28 +2,12 @@ package net.sergeych.lynon
 
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.Obj
-import net.sergeych.lyng.obj.ObjBool
-import net.sergeych.lyng.obj.ObjChar
-import net.sergeych.lyng.obj.ObjInt
 
-class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput()) : LynonEncoder(bout) {
+open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonSettings.default) {
-    fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
-}
-
-class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
-
-open class LynonEncoder(val bout: BitOutput) {
-
-    fun shouldCache(obj: Obj): Boolean = when (obj) {
-        is ObjChar -> false
-        is ObjInt -> obj.value > 0x10000FF
-        is ObjBool -> false
-        else -> true
-    }
 
     val cache = mutableMapOf<Any, Int>()
 
-    inline fun encodeCached(item: Any, packer: LynonEncoder.() -> Unit) {
+    private inline fun encodeCached(item: Any, packer: LynonEncoder.() -> Unit) {
         if (item is Obj) {
             cache[item]?.let { cacheId ->
                 val size = sizeInBits(cache.size)
@@ -31,11 +15,8 @@ open class LynonEncoder(val bout: BitOutput) {
                 bout.putBits(cacheId.toULong(), size)
             } ?: run {
                 bout.putBit(0)
-                if (shouldCache(item)) {
+                if (settings.shouldCache(item))
-                    bout.putBit(1)
                     cache[item] = cache.size
-                } else
-                    bout.putBit(0)
                 packer()
             }
         }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonSettings.kt

@@ -0,0 +1,20 @@
+package net.sergeych.lynon
+
+import net.sergeych.lyng.obj.Obj
+import net.sergeych.lyng.obj.ObjBool
+import net.sergeych.lyng.obj.ObjChar
+import net.sergeych.lyng.obj.ObjInt
+
+open class LynonSettings() {
+
+    open fun shouldCache(obj: Obj): Boolean = when (obj) {
+        is ObjChar -> false
+        is ObjInt -> obj.value > 0x10000FF
+        is ObjBool -> false
+        else -> true
+    }
+
+    companion object {
+        val default = LynonSettings()
+    }
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw0.kt

@@ -0,0 +1,96 @@
+package net.sergeych.lynon
+
+import net.sergeych.bintools.ByteChunk
+import kotlin.math.roundToInt
+
+/**
+ * LZW compression algorithm: work in progress.
+ *
+ * Uses Lyng but input/output. Uses automatic code size.
+ *
+ * TODO: - reset dictionary
+ */
+class LZW {
+    companion object {
+
+        val MAX_CODE_SIZE = 12
+        val STOP_CODE = (1 shl MAX_CODE_SIZE) - 1
+        val MAX_DICT_SIZE = (STOP_CODE * 0.92).roundToInt()
+
+
+        /**
+         * Compresses the input string using LZW algorithm
+         * @param input The string to compress
+         * @return List of compressed codes
+         */
+        fun compress(input: UByteArray,bitOutput: BitOutput) {
+            // Initialize dictionary with all possible single characters
+            val dictionary = mutableMapOf<ByteChunk, Int>()
+            for (i in 0..255) {
+                // 23
+                dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
+            }
+
+            var nextCode = 256
+            var current = ByteChunk(ubyteArrayOf())
+//            val result = mutableListOf<Int>()
+
+            for (char in input) {
+                val combined = current + char
+                if (dictionary.containsKey(combined)) {
+                    current = combined
+                } else {
+                    val size = sizeInBits(dictionary.size)
+                    bitOutput.putBits(dictionary[current]!!,size)
+                    dictionary[combined] = nextCode++
+                    current = ByteChunk(ubyteArrayOf(char))
+                }
+            }
+
+            if (current.size > 0) {
+                val size = sizeInBits(dictionary.size)
+                bitOutput.putBits(dictionary[current]!!,size)
+            }
+        }
+
+        /**
+         * Decompresses a list of LZW codes back to the original string
+         * @param compressed The list of compressed codes
+         * @return The decompressed string
+         */
+        fun decompress(compressed: BitInput): UByteArray {
+            // Initialize dictionary with all possible single characters
+            val dictionary = mutableMapOf<Int, UByteArray>()
+            for (i in 0..255) {
+                dictionary[i] = ubyteArrayOf(i.toUByte())
+            }
+
+            var nextCode = 256
+            var previous = dictionary[compressed.getBits(9).toInt()]!!
+            val result = mutableListOf<UByte>()
+
+            while( !compressed.isEndOfStream ) {
+                val codeSize = sizeInBits(nextCode + 1)
+                val code = compressed.getBitsOrNull(codeSize)?.toInt() ?: break
+                val current = if ( code in dictionary) {
+                    dictionary[code]!!
+                } else if (code == nextCode) {
+                    // Special case for pattern like cScSc
+                    previous + previous[0]
+                } else {
+                    throw IllegalArgumentException("Invalid compressed code: $code")
+                }
+
+                result += current
+                dictionary[nextCode++] = previous + current[0]
+                previous = current
+            }
+
+            return result.toTypedArray().toUByteArray()
+        }
+    }
+}
+
+private operator fun ByteChunk.plus(byte: UByte): ByteChunk {
+    return ByteChunk(data + byte)
+}

lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt

@@ -0,0 +1,14 @@
+package net.sergeych.lynon
+
+/**
+ * Variant of [LynonEncoder] that writes to embedded [MemoryBitOutput]
+ */
+class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput(), settings: LynonSettings = LynonSettings.default)
+    : LynonEncoder(bout, settings) {
+    fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
+}
+
+/**
+ * Variant of [LynonDecoder] that reads from a given `source` using [MemoryBitInput]
+ */
+class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))

lynglib/src/jvmTest/kotlin/LynonTests.kt

@@ -3,6 +3,8 @@ import kotlinx.coroutines.test.runTest
 import net.sergeych.lyng.Scope
 import net.sergeych.lyng.obj.*
 import net.sergeych.lynon.*
+import java.nio.file.Files
+import java.nio.file.Path
 import kotlin.test.Test
 
 class LynonTests {
@@ -189,4 +191,27 @@ class LynonTests {
         assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
         assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
     }
+
+    @Test
+    fun testLzw() {
+        // Example usage
+//        val original = "TOBEORNOTTOBEORTOBEORNOT"
+        val original = Files.readString(Path.of("../sample_texts/dikkens_hard_times.txt"))
+//        println("Original: $original")
+        println("Length: ${original.length}")
+
+        // Compress
+        val out = MemoryBitOutput()
+        LZW.compress(original.encodeToByteArray().toUByteArray(), out)
+//        println("\nCompressed codes: ${out.toUByteArray().toDump()}")
+        println("Number of codes: ${out.toUByteArray().size}")
+
+//        // Decompress
+        val decompressed = LZW.decompress(MemoryBitInput(out.toUByteArray())).toByteArray().decodeToString()
+//        println("\nDecompressed: $decompressed")
+        println("Length: ${decompressed.length}")
+
+        // Verification
+        println("\nOriginal and decompressed match: ${original == decompressed}")
+    }
 }