diff --git a/.gitignore b/.gitignore index 9359b89..0465c33 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ xcuserdata .gigaide /kotlin-js-store/yarn.lock /test.lyng +/sample_texts/1.txt.gz diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt index a30fe02..972595f 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/Obj.kt @@ -366,6 +366,7 @@ object ObjNull : Obj() { override suspend fun lynonType(): LynonType { return LynonType.Null } + override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) { if (lynonType == null) { encoder.putBit(0) diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt index a02ef6e..8296e1d 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjBuffer.kt @@ -82,9 +82,7 @@ open class ObjBuffer(val byteArray: UByteArray) : Obj() { override suspend fun lynonType(): LynonType = LynonType.Buffer override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) { - encoder.encodeCached(byteArray) { - bout.compress(byteArray.asByteArray()) - } + encoder.encodeCachedBytes(byteArray.asByteArray()) } companion object { diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt index 6dbd327..b3068a0 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjInt.kt @@ -117,6 +117,7 @@ class ObjInt(var value: Long, override val isConst: Boolean = false) : Obj(), Nu LynonType.Int0 -> {} LynonType.IntPositive -> encoder.encodeUnsigned(value.toULong()) LynonType.IntNegative -> encoder.encodeUnsigned((-value).toULong()) + LynonType.IntSigned -> encoder.encodeSigned(value) else -> scope.raiseIllegalArgument("Unsupported lynon type code for Int: $lynonType") } } @@ -131,6 +132,7 @@ class ObjInt(var value: Long, override val isConst: Boolean = false) : Obj(), Nu LynonType.Int0 -> Zero LynonType.IntPositive -> ObjInt(decoder.unpackUnsigned().toLong()) LynonType.IntNegative -> ObjInt(-decoder.unpackUnsigned().toLong()) + LynonType.IntSigned -> ObjInt(decoder.unpackSigned()) else -> scope.raiseIllegalState("illegal type code for Int: $lynonType") } } diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt index 93a9816..d3a1339 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjList.kt @@ -2,6 +2,9 @@ package net.sergeych.lyng.obj import net.sergeych.lyng.Scope import net.sergeych.lyng.statement +import net.sergeych.lynon.LynonDecoder +import net.sergeych.lynon.LynonEncoder +import net.sergeych.lynon.LynonType class ObjList(val list: MutableList = mutableListOf()) : Obj() { @@ -125,9 +128,18 @@ class ObjList(val list: MutableList = mutableListOf()) : Obj() { return list == other.list } - companion object { - val type = ObjClass("List", ObjArray).apply { + override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) { + encoder.encodeAnyList(scope,list) + } + override suspend fun lynonType(): LynonType = LynonType.List + + companion object { + val type = object : ObjClass("List", ObjArray) { + override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj { + return ObjList(decoder.decodeAnyList(scope)) + } + }.apply { createField("size", statement { (thisObj as ObjList).list.size.toObj() diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt index 8e8f884..ba75e0e 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lyng/obj/ObjString.kt @@ -84,17 +84,14 @@ data class ObjString(val value: String) : Obj() { override suspend fun lynonType(): LynonType = LynonType.String override suspend fun serialize(scope: Scope, encoder: LynonEncoder, lynonType: LynonType?) { - val data = value.encodeToByteArray() - encoder.encodeCached(data) { encoder.encodeBinaryData(data) } + encoder.encodeBinaryData(value.encodeToByteArray()) } companion object { val type = object : ObjClass("String") { override suspend fun deserialize(scope: Scope, decoder: LynonDecoder, lynonType: LynonType?): Obj = - decoder.decodeCached { - ObjString(decoder.unpackBinaryData().decodeToString()) - } + ObjString(decoder.unpackBinaryData().decodeToString()) }.apply { addFn("toInt") { ObjInt(thisAs().value.toLong()) diff --git a/lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt b/lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt index cef2da4..2915fd3 100644 --- a/lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt +++ b/lynglib/src/commonMain/kotlin/net/sergeych/lynon/LynonDecoder.kt @@ -17,10 +17,11 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe val cache = mutableListOf() - inline fun decodeCached(f: LynonDecoder.() -> T): T { + inline fun decodeCached(f: LynonDecoder.() -> T): T { return if (bin.getBit() == 0) { // unpack and cache f().also { +// println("decode: cache miss: ${cache.size}: $it:${it::class.simpleName}") if (settings.shouldCache(it)) cache.add(it) } } else { @@ -29,7 +30,8 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe val id = bin.getBitsOrNull(size)?.toInt() ?: throw RuntimeException("Invalid object id: unexpected end of stream") if (id >= cache.size) throw RuntimeException("Invalid object id: $id should be in 0..<${cache.size}") - @Suppress("UNCHECKED_CAST") +// println("decode: cache hit ${id}: ${cache[id]}:${cache[id]::class.simpleName}") +// @Suppress("UNCHECKED_CAST") cache[id] as T } } @@ -39,8 +41,29 @@ open class LynonDecoder(val bin: BitInput, val settings: LynonSettings = LynonSe type.objClass.deserialize(scope, this, type) } - suspend fun decodeObject(scope: Scope, type: ObjClass): Obj { - return decodeCached { type.deserialize(scope, this, null) } + suspend fun decodeAnyList(scope: Scope): MutableList { + return if( bin.getBit() == 1) { + // homogenous + val type = LynonType.entries[getBitsAsInt(4)] + val size = bin.unpackUnsigned().toInt() + println("detected homogenous list type $type, $size items") + val list = mutableListOf() + val objClass = type.objClass + for( i in 0 ..< size) { + list += decodeObject(scope, objClass, type).also { + println("decoded: $it") + } + } + list + } + else { + val size = unpackUnsigned().toInt() + (0.. other // upgrade 0 to some other int + other == Int0 -> this // 0 is member of our class, ignore + // different signum propagate to signed + else -> IntSigned + } + } else + // impossible to generalize + null + ).also { println("Gen $this + $other -> $it") } + } + + val isInt by lazy { + when (this) { + Int0, IntSigned, IntPositive, IntNegative -> true + else -> false + } + } + } open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = LynonSettings.default) { @@ -29,14 +53,20 @@ open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = Lynon suspend fun serializeAndCache(key: Any = item) { cache[key]?.let { cacheId -> +// println("encode: Cache hit: ${cacheId}: $item: ${item::class.simpleName}") val size = sizeInBits(cache.size) bout.putBit(1) bout.putBits(cacheId.toULong(), size) } ?: run { bout.putBit(0) - if (settings.shouldCache(item)) + if (settings.shouldCache(item)) { +// println("encode add cache: ${cache.size}: $item: ${item::class.simpleName}") + packer() cache[key] = cache.size - packer() + } else { +// println("encode but not cache $item") + packer() + } } } @@ -53,21 +83,66 @@ open class LynonEncoder(val bout: BitOutput, val settings: LynonSettings = Lynon * * Caching is used automatically. */ - suspend fun encodeAny(scope: Scope, value: Obj) { - encodeCached(value) { - val type = value.lynonType() - putType(type) - value.serialize(scope, this, type) + suspend fun encodeAny(scope: Scope, obj: Obj) { + encodeCached(obj) { + val type = putTypeRecord(obj, obj.lynonType()) + obj.serialize(scope, this, type) } } + private fun putTypeRecord(obj: Obj, type: LynonType): LynonType { + putType(type) + return type + } + private fun putType(type: LynonType) { bout.putBits(type.ordinal.toULong(), 4) } - suspend fun encodeObject(scope: Scope, obj: Obj) { + /** + * AnyList could be homogenous (first bit=1) and heterogeneous (first bit=0). Homogenous list + * has a single type record that precedes the list, heterogeneous hash typed record + * for each item. + * + */ + suspend fun encodeAnyList(scope: Scope, list: List) { + val objClass = list[0].objClass + var type = list[0].lynonType() + var isHomogeneous = true + for (i in list.drop(1)) + if (i.objClass != objClass) { + isHomogeneous = false + break + } else { + // same class but type might need generalization + type = type.generalizeTo(i.lynonType()) + ?: scope.raiseError("inner error: can't generalize lynon type $type to ${i.lynonType()}") + } + if (isHomogeneous) { + putBit(1) + putTypeRecord(list[0], type) + encodeUnsigned(list.size.toULong()) + for (i in list) encodeObject(scope, i, type) + } else { + putBit(0) + encodeUnsigned(list.size.toULong()) + for (i in list) encodeAny(scope, i) + } + + } + + /** + * Write object _with no type record_: type is known + */ + suspend fun encodeObject(scope: Scope, obj: Obj,overrideType: LynonType? = null) { encodeCached(obj) { - obj.serialize(scope, this, null) + obj.serialize(scope, this, overrideType) + } + } + + suspend fun encodeCachedBytes(bytes: ByteArray) { + encodeCached(bytes) { + bout.compress(bytes) } } diff --git a/lynglib/src/jvmTest/kotlin/LynonTests.kt b/lynglib/src/jvmTest/kotlin/LynonTests.kt index da762e2..e76a17d 100644 --- a/lynglib/src/jvmTest/kotlin/LynonTests.kt +++ b/lynglib/src/jvmTest/kotlin/LynonTests.kt @@ -1,4 +1,5 @@ -import junit.framework.TestCase.* +import junit.framework.TestCase.assertNotSame +import junit.framework.TestCase.assertSame import kotlinx.coroutines.test.runTest import net.sergeych.bintools.encodeToHex import net.sergeych.lyng.Scope @@ -9,6 +10,8 @@ import java.nio.file.Files import java.nio.file.Path import kotlin.test.Test import kotlin.test.assertContentEquals +import kotlin.test.assertEquals + class LynonTests { @Test @@ -144,6 +147,15 @@ class LynonTests { assertEquals(1471792L, bin.unpackSigned()) } + @Test + fun testObjStringAndStringKeys() = runTest { + val s = "foo" + val sobj = ObjString("foo") + val map = mutableMapOf(s to 1, sobj to 2) + assertEquals(1, map[s]) + assertEquals(2, map[sobj]) + } + @Test fun testCache1() = runTest { val bout = MemoryBitOutput() @@ -398,14 +410,12 @@ class LynonTests { val alphabet = object : Huffman.Alphabet { override val maxOrdinal = LynonType.entries.size -// val bitSize = sizeInBits(maxOrdinal) - override fun decodeOrdinalTo(bout: BitOutput, ordinal: Int) { - TODO("Not yet implemented") + throw NotImplementedError() } override fun get(ordinal: Int): LynonType { - TODO("Not yet implemented") + return LynonType.entries[ordinal] } override fun ordinalOf(value: LynonType): Int = value.ordinal @@ -474,5 +484,19 @@ class LynonTests { assertEquals(src3, bin.decompressString()) } + @Test + fun testIntList() = runTest { + testScope().eval(""" +// testEncode([1,2,3]) +// testEncode([-1,-2,-3]) +// testEncode([1,-2,-3]) +// testEncode([0,1]) +// testEncode([0,0,0]) +// testEncode(["the", "the", "wall", "the", "wall", "wall"]) + testEncode([1,2,3, "the", "wall", "wall"]) + + """.trimIndent()) + } + }