From 63b231975bb610527639674c7c813048f8908720 Mon Sep 17 00:00:00 2001 From: sergeych Date: Sun, 12 Mar 2023 22:48:21 +0100 Subject: [PATCH] encoder and decoder now support extendable formats. --- build.gradle.kts | 24 +++++++ .../kotlin/net.sergeych.bintools/DataSink.kt | 29 +++++++- .../net.sergeych.bintools/DataSource.kt | 29 ++++++-- .../kotlin/net.sergeych.bintools/IntCodec.kt | 7 +- .../net.sergeych.bintools/simple_codecs.kt | 19 ++++++ .../kotlin/net.sergeych.bintools/smartint.kt | 2 +- .../kotlin/net.sergeych.bintools/varint.kt | 11 ++- .../net.sergeych.bipack/BipackDecoder.kt | 68 +++++++++++++++++++ .../net.sergeych.bipack/BipackEncoder.kt | 66 ++++++++++++++++++ .../kotlin/net.sergeych.bipack/Extendable.kt | 18 +++++ .../kotlin/bintools/SmartintTest.kt | 6 +- src/commonTest/kotlin/bintools/VarintTest.kt | 2 +- .../net/sergeych/bipack/BipackEncoderTest.kt | 30 ++++++++ 13 files changed, 288 insertions(+), 23 deletions(-) create mode 100644 src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt create mode 100644 src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt create mode 100644 src/commonMain/kotlin/net.sergeych.bipack/Extendable.kt create mode 100644 src/jvmTest/kotlin/net/sergeych/bipack/BipackEncoderTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index d53e8ca..bcacbb0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,7 @@ plugins { kotlin("multiplatform") version "1.8.10" + kotlin("plugin.serialization") version "1.8.10" + `maven-publish` } group = "net.sergeych" @@ -39,6 +41,9 @@ kotlin { sourceSets { + all { + languageSettings.optIn("kotlinx.serialization.ExperimentalSerializationApi") + } val commonMain by getting { dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") @@ -57,4 +62,23 @@ kotlin { val nativeMain by getting val nativeTest by getting } + + publishing { + val mavenToken by lazy { + File("${System.getProperty("user.home")}/.gitea_token").readText() + } + repositories { + maven { + credentials(HttpHeaderCredentials::class) { + name = "Authorization" + value = mavenToken + } + url = uri("https://gitea.sergeych.net/api/packages/SergeychWorks/maven") + authentication { + create("Authorization", HttpHeaderAuthentication::class) + } + } + } + } + } diff --git a/src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt b/src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt index 28c8af8..7c93b29 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt @@ -6,16 +6,41 @@ interface DataSink { fun writeByte(data: Int) = writeByte(data.toByte()) - open fun writeUByte(data: UByte) { + fun writeUByte(data: UByte) { writeByte(data.toByte()) } + fun writeDouble(value: Double) { + writeI64(value.toRawBits()) + } + + fun writeFloat(value: Float) { + writeI32(value.toRawBits()) + } + @Suppress("unused") - open fun writeBytes(data: ByteArray) { + fun writeBytes(data: ByteArray) { for(d in data) writeByte(d) } } +inline fun DataSink.writeNumber(value: T) { + when(value) { + is Float -> writeFloat(value) + is Double -> writeDouble(value) + else -> Smartint.encode(value, this) + } +} + +fun DataSink.writeI32(value: Int) { + writeBytes(intToBytes(value)) +} + +fun DataSink.writeI64(value: Long) { + writeBytes(longToBytes(value)) +} + + class ArrayDataSink : DataSink { private val result = mutableListOf() diff --git a/src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt b/src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt index 91fc31a..3e9b948 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt @@ -1,5 +1,7 @@ package net.sergeych.bintools +import kotlin.reflect.typeOf + /** * data input stream-like abstraction. We need it because * kotlinx serialization is synchronous and there us nothing @@ -8,23 +10,40 @@ package net.sergeych.bintools */ interface DataSource { + class EndOfData() : Exception("no more data available") + fun readByte(): Byte - val position: Int fun readUByte() = readByte().toUByte() @Suppress("unused") fun readBytes(size: Int): ByteArray = ByteArray(size).also { a -> - for( i in 0..size) + for (i in 0 until size) a[i] = readByte() } + + fun readI32(): Int = bytesToInt(readBytes(4)) + fun readI64(): Long = bytesToLong(readBytes(8)) + + fun readDouble() = Double.fromBits(readI64()) + fun readFloat() = Float.fromBits(readI32()) + } fun ByteArray.toDataSource(): DataSource = object : DataSource { - override var position = 0 + var position = 0 + private set - override fun readByte(): Byte = this@toDataSource[position++] - } \ No newline at end of file + override fun readByte(): Byte = + if( position < size ) this@toDataSource[position++] + else throw DataSource.EndOfData() + } + +inline fun DataSource.readNumber(): T = when(typeOf()) { + typeOf() -> readDouble() as T + typeOf() -> readFloat() as T + else -> Smartint.decode(this) +} diff --git a/src/commonMain/kotlin/net.sergeych.bintools/IntCodec.kt b/src/commonMain/kotlin/net.sergeych.bintools/IntCodec.kt index ce0528f..4f59f46 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/IntCodec.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/IntCodec.kt @@ -1,6 +1,5 @@ package net.sergeych.bintools -import com.icodici.ubdata.Varint import kotlin.reflect.typeOf /** @@ -17,7 +16,7 @@ interface IntCodec { /** * Default signed codec uses bit 0 as a sign (to keep packed as small as possible) */ - fun encodeSigned(value: Long, sink: DataSink): Unit { + fun encodeSigned(value: Long, sink: DataSink) { var sigBit: ULong var x: ULong if (value < 0) { @@ -62,9 +61,11 @@ inline fun IntCodec.decode(source: DataSource): T { return when (typeOf()) { typeOf() -> decodeUnsigned(source).toUByte() typeOf() -> decodeUnsigned(source).toUInt() + typeOf() -> decodeUnsigned(source).toUShort() typeOf() -> decodeUnsigned(source).toULong() typeOf() -> decodeSigned(source).toByte() typeOf() -> decodeSigned(source).toInt() + typeOf() -> decodeSigned(source).toShort() typeOf() -> decodeSigned(source).toLong() else -> throw IllegalArgumentException("can't decode to ${T::class.simpleName}") @@ -75,9 +76,11 @@ inline fun IntCodec.encode(x: T, dout: DataSink) { when (x) { is UByte -> encodeUnsigned(x.toULong(), dout) is UInt -> encodeUnsigned(x.toULong(), dout) + is UShort -> encodeUnsigned(x.toULong(), dout) is ULong -> encodeUnsigned(x, dout) is Byte -> encodeSigned(x.toLong(), dout) is Int -> encodeSigned(x.toLong(), dout) + is Short -> encodeSigned(x.toLong(), dout) is Long -> encodeSigned(x, dout) else -> throw IllegalArgumentException("can't encode with varitn ${x::class.simpleName}: $x") } diff --git a/src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt b/src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt index 6d599cc..c2f3c1b 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt @@ -11,6 +11,16 @@ fun longToBytes(value: Long): ByteArray { return result } +fun intToBytes(value: Int): ByteArray { + var l = value + val result = ByteArray(8) + for (i in 3 downTo 0) { + result[i] = (l and 0xFF).toByte() + l = l shr 8 + } + return result +} + /** * Convert 8 bytes to LE long */ @@ -23,6 +33,15 @@ fun bytesToLong(b: ByteArray): Long { return result } +fun bytesToInt(b: ByteArray): Int { + var result: Int = 0 + for (i in 0 until 4) { + result = result shl 8 + result = result or (b[i].toInt() and 0xFF) + } + return result +} + private val hexDigits = "0123456789ABCDEF" diff --git a/src/commonMain/kotlin/net.sergeych.bintools/smartint.kt b/src/commonMain/kotlin/net.sergeych.bintools/smartint.kt index 424672e..71fd8c7 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/smartint.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/smartint.kt @@ -1,6 +1,6 @@ @file:OptIn(ExperimentalUnsignedTypes::class) -package com.icodici.ubdata +package net.sergeych.bintools import net.sergeych.bintools.* diff --git a/src/commonMain/kotlin/net.sergeych.bintools/varint.kt b/src/commonMain/kotlin/net.sergeych.bintools/varint.kt index 7aef94e..fdddb3a 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/varint.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/varint.kt @@ -1,8 +1,5 @@ -@file:OptIn(ExperimentalUnsignedTypes::class) -package com.icodici.ubdata - -import net.sergeych.bintools.* +package net.sergeych.bintools /** * Variable-length long integer encoding. the MSB (0x80) bit of each byte flags @@ -14,15 +11,15 @@ import net.sergeych.bintools.* * same or even worse, see [Smartint] docs. */ object Varint: IntCodec { - override fun encodeUnsigned(value: ULong, dout: DataSink) { + override fun encodeUnsigned(value: ULong, sink: DataSink) { var rest = value do { val x = (rest and 127u).toInt() rest = rest shr 7 if (rest > 0u) - dout.writeByte(x or 0x80) + sink.writeByte(x or 0x80) else - dout.writeByte(x) + sink.writeByte(x) } while (rest > 0u) } diff --git a/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt b/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt new file mode 100644 index 0000000..32b47d9 --- /dev/null +++ b/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt @@ -0,0 +1,68 @@ +package net.sergeych.bipack + +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.AbstractDecoder +import kotlinx.serialization.encoding.CompositeDecoder +import kotlinx.serialization.modules.EmptySerializersModule +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.serializer +import net.sergeych.bintools.DataSource +import net.sergeych.bintools.readNumber +import net.sergeych.bintools.toDataSource + +class BipackDecoder(val input: DataSource, var elementsCount: Int = 0) : AbstractDecoder() { + private var elementIndex = 0 + + override val serializersModule: SerializersModule = EmptySerializersModule + override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0 + override fun decodeByte(): Byte = input.readByte() + override fun decodeShort(): Short = input.readNumber() + override fun decodeInt(): Int = input.readNumber() + override fun decodeLong(): Long = input.readNumber() + override fun decodeFloat(): Float = input.readFloat() + override fun decodeDouble(): Double = input.readDouble() + override fun decodeChar(): Char = Char(input.readNumber().toInt()) + + fun readBytes(): ByteArray { + val length = input.readNumber() + return input.readBytes(length.toInt()) + } + + override fun decodeString(): String = readBytes().decodeToString() + override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readNumber().toInt() + + override fun decodeElementIndex(descriptor: SerialDescriptor): Int { + if (elementIndex >= elementsCount) return CompositeDecoder.DECODE_DONE + return elementIndex++ + } + + override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { + return BipackDecoder( + input, + if (descriptor.annotations.any { it is ExtendableFormat }) + input.readNumber().toInt() + else descriptor.elementsCount + ) + } + + override fun decodeCollectionSize(descriptor: SerialDescriptor): Int = + input.readNumber().toInt().also { + elementsCount = it + } + + override fun decodeNotNullMark(): Boolean = decodeBoolean() + + @ExperimentalSerializationApi + override fun decodeNull(): Nothing? = null + + companion object { + fun decode(source: DataSource, deserializer: DeserializationStrategy): T = + BipackDecoder(source).decodeSerializableValue(deserializer) + + inline fun decode(source: DataSource): T = decode(source, serializer()) + inline fun decode(source: ByteArray): T = + decode(source.toDataSource(), serializer()) + } +} diff --git a/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt b/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt new file mode 100644 index 0000000..6ae0691 --- /dev/null +++ b/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt @@ -0,0 +1,66 @@ +package net.sergeych.bipack + +import kotlinx.serialization.SerializationStrategy +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.AbstractEncoder +import kotlinx.serialization.encoding.CompositeEncoder +import kotlinx.serialization.modules.EmptySerializersModule +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.serializer +import net.sergeych.bintools.ArrayDataSink +import net.sergeych.bintools.DataSink +import net.sergeych.bintools.writeI64 +import net.sergeych.bintools.writeNumber + +class BipackEncoder(val output: DataSink) : AbstractEncoder() { + override val serializersModule: SerializersModule = EmptySerializersModule + override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0) + override fun encodeByte(value: Byte) = output.writeByte(value.toInt()) + override fun encodeShort(value: Short) = output.writeNumber(value.toInt()) + override fun encodeInt(value: Int) = output.writeNumber(value) + + fun encodeUInt(value: UInt) = output.writeNumber(value) + override fun encodeLong(value: Long) = output.writeNumber(value) + override fun encodeFloat(value: Float) = output.writeNumber(value) + override fun encodeDouble(value: Double) = output.writeI64(value.toRawBits()) + override fun encodeChar(value: Char) = output.writeNumber(value.code.toUInt()) + override fun encodeString(value: String) { +// output.writeUTF(value) + writeBytes(value.encodeToByteArray()) + } + + fun writeBytes(value: ByteArray) { + output.writeNumber(value.size.toUInt()) + output.writeBytes(value) + } + + override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeNumber(index.toUInt()) + + override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder { + encodeUInt(collectionSize.toUInt()) + return this + } + + override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { + if( descriptor.annotations.any { it is ExtendableFormat } ) + encodeUInt(descriptor.elementsCount.toUInt()) + return super.beginStructure(descriptor) + } + + override fun encodeNull() = encodeBoolean(false) + override fun encodeNotNullMark() = encodeBoolean(true) + + companion object { + fun encode(serializer: SerializationStrategy, value: T,sink: DataSink) { + val encoder = BipackEncoder(sink) + encoder.encodeSerializableValue(serializer, value) + } + + fun encode(serializer: SerializationStrategy, value: T): ByteArray = + ArrayDataSink().also { encode(serializer, value, it)}.toByteArray() + + inline fun encode(value: T) = encode(serializer(), value) + inline fun encode(value: T,sink: DataSink) = encode(serializer(), value, sink) + + } +} diff --git a/src/commonMain/kotlin/net.sergeych.bipack/Extendable.kt b/src/commonMain/kotlin/net.sergeych.bipack/Extendable.kt new file mode 100644 index 0000000..24d99d5 --- /dev/null +++ b/src/commonMain/kotlin/net.sergeych.bipack/Extendable.kt @@ -0,0 +1,18 @@ +package net.sergeych.bipack + +import kotlinx.serialization.SerialInfo + +/** + * If this annotation is presented in some @Serializable class defition, its instances + * will be serialized with leadinf number of fields. This allows to extend class later + * providing new parameters __to the end of the class__ and _with default values__. + * + * Whe deserializing such instances from previous version binaries, the new parameters + * will get default values. + * + * Serialized data of classes not market as ExtendableFormat could not be changed without + * breaking compatibility with existing serialized data. + */ +@Target(AnnotationTarget.CLASS) +@SerialInfo +annotation class ExtendableFormat diff --git a/src/commonTest/kotlin/bintools/SmartintTest.kt b/src/commonTest/kotlin/bintools/SmartintTest.kt index 5f570ae..febb8cd 100644 --- a/src/commonTest/kotlin/bintools/SmartintTest.kt +++ b/src/commonTest/kotlin/bintools/SmartintTest.kt @@ -1,10 +1,6 @@ package bintools -import com.icodici.ubdata.Smartint -import com.icodici.ubdata.Varint -import net.sergeych.bintools.decode -import net.sergeych.bintools.encode -import net.sergeych.bintools.encodeToHex +import net.sergeych.bintools.* import kotlin.test.Test import kotlin.test.assertEquals diff --git a/src/commonTest/kotlin/bintools/VarintTest.kt b/src/commonTest/kotlin/bintools/VarintTest.kt index 258628d..7c15671 100644 --- a/src/commonTest/kotlin/bintools/VarintTest.kt +++ b/src/commonTest/kotlin/bintools/VarintTest.kt @@ -1,6 +1,6 @@ package bintools -import com.icodici.ubdata.Varint +import net.sergeych.bintools.Varint import net.sergeych.bintools.decode import net.sergeych.bintools.encode import net.sergeych.bintools.encodeToHex diff --git a/src/jvmTest/kotlin/net/sergeych/bipack/BipackEncoderTest.kt b/src/jvmTest/kotlin/net/sergeych/bipack/BipackEncoderTest.kt new file mode 100644 index 0000000..3e97311 --- /dev/null +++ b/src/jvmTest/kotlin/net/sergeych/bipack/BipackEncoderTest.kt @@ -0,0 +1,30 @@ +package net.sergeych.bipack + +import kotlinx.serialization.Serializable +import net.sergeych.bintools.toDump +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +@Serializable +@ExtendableFormat +data class Foobar1(val bar: Int, val foo: Int = 117) + +@Serializable +@ExtendableFormat +data class Foobar2(val bar: Int, val foo: Int,val other: Int = -1) + +class BipackEncoderTest { + + @Test + fun encodeSimple() { + val a = Foobar1(42)//, "bum") + println(BipackEncoder.encode(a).toDump()) + val b = BipackDecoder.decode(BipackEncoder.encode(a)) + assertEquals(a, b) + val c = BipackDecoder.decode(BipackEncoder.encode(a)) + assertEquals(-1, c.other) +// assertEquals(a.foo, c.foo) + assertEquals(a.bar, c.bar) +// assertEquals(a.buzz, c.buzz) + } +} \ No newline at end of file