encoder and decoder now support extendable formats.

build.gradle.kts

@@ -1,5 +1,7 @@
 plugins {
     kotlin("multiplatform") version "1.8.10"
+    kotlin("plugin.serialization") version "1.8.10"
+    `maven-publish`
 }
 
 group = "net.sergeych"
@@ -39,6 +41,9 @@ kotlin {
 
     
     sourceSets {
+        all {
+            languageSettings.optIn("kotlinx.serialization.ExperimentalSerializationApi")
+        }
         val commonMain by getting {
             dependencies {
                 implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4")
@@ -57,4 +62,23 @@ kotlin {
         val nativeMain by getting
         val nativeTest by getting
     }
+
+    publishing {
+        val mavenToken by lazy {
+            File("${System.getProperty("user.home")}/.gitea_token").readText()
+        }
+        repositories {
+            maven {
+                credentials(HttpHeaderCredentials::class) {
+                    name = "Authorization"
+                    value = mavenToken
+                }
+                url = uri("https://gitea.sergeych.net/api/packages/SergeychWorks/maven")
+                authentication {
+                    create("Authorization", HttpHeaderAuthentication::class)
+                }
+            }
+        }
+    }
+
 }

src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt

@@ -6,16 +6,41 @@ interface DataSink {
 
     fun writeByte(data: Int) = writeByte(data.toByte())
 
-    open fun writeUByte(data: UByte) {
+    fun writeUByte(data: UByte) {
         writeByte(data.toByte())
     }
 
+    fun writeDouble(value: Double) {
+        writeI64(value.toRawBits())
+    }
+
+    fun writeFloat(value: Float) {
+        writeI32(value.toRawBits())
+    }
+
     @Suppress("unused")
-    open fun writeBytes(data: ByteArray) {
+    fun writeBytes(data: ByteArray) {
         for(d in data) writeByte(d)
     }
 }
 
+inline fun <reified T:Any>DataSink.writeNumber(value: T) {
+    when(value) {
+        is Float -> writeFloat(value)
+        is Double -> writeDouble(value)
+        else -> Smartint.encode(value, this)
+    }
+}
+
+fun DataSink.writeI32(value: Int) {
+    writeBytes(intToBytes(value))
+}
+
+fun DataSink.writeI64(value: Long) {
+    writeBytes(longToBytes(value))
+}
+
+
 class ArrayDataSink : DataSink {
     private val result = mutableListOf<Byte>()
 

src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt

@@ -1,5 +1,7 @@
 package net.sergeych.bintools
 
+import kotlin.reflect.typeOf
+
 /**
  * data input stream-like abstraction. We need it because
  * kotlinx serialization is synchronous and there us nothing
@@ -8,23 +10,40 @@ package net.sergeych.bintools
  */
 interface DataSource {
 
+    class EndOfData() : Exception("no more data available")
+
     fun readByte(): Byte
 
-    val position: Int
 
     fun readUByte() = readByte().toUByte()
 
     @Suppress("unused")
     fun readBytes(size: Int): ByteArray =
         ByteArray(size).also { a ->
-            for( i in 0..size)
+            for (i in 0 until size)
                 a[i] = readByte()
         }
+
+    fun readI32(): Int = bytesToInt(readBytes(4))
+    fun readI64(): Long = bytesToLong(readBytes(8))
+
+    fun readDouble() = Double.fromBits(readI64())
+    fun readFloat() = Float.fromBits(readI32())
+
 }
 
 fun ByteArray.toDataSource(): DataSource =
     object : DataSource {
-        override var position = 0
+        var position = 0
+            private set
 
-        override fun readByte(): Byte = this@toDataSource[position++]
+        override fun readByte(): Byte =
+            if( position < size ) this@toDataSource[position++]
+        else throw DataSource.EndOfData()
     }
+
+inline fun <reified T : Any> DataSource.readNumber(): T = when(typeOf<T>()) {
+    typeOf<Double>() -> readDouble() as T
+    typeOf<Float>() -> readFloat() as T
+    else -> Smartint.decode(this)
+}

src/commonMain/kotlin/net.sergeych.bintools/IntCodec.kt

@@ -1,6 +1,5 @@
 package net.sergeych.bintools
 
-import com.icodici.ubdata.Varint
 import kotlin.reflect.typeOf
 
 /**
@@ -17,7 +16,7 @@ interface IntCodec {
     /**
      * Default signed codec uses bit 0 as a sign (to keep packed as small as possible)
      */
-    fun encodeSigned(value: Long, sink: DataSink): Unit {
+    fun encodeSigned(value: Long, sink: DataSink) {
         var sigBit: ULong
         var x: ULong
         if (value < 0) {
@@ -62,9 +61,11 @@ inline fun <reified T : Any> IntCodec.decode(source: DataSource): T {
     return when (typeOf<T>()) {
         typeOf<UByte>() -> decodeUnsigned(source).toUByte()
         typeOf<UInt>() -> decodeUnsigned(source).toUInt()
+        typeOf<UShort>() -> decodeUnsigned(source).toUShort()
         typeOf<ULong>() -> decodeUnsigned(source).toULong()
         typeOf<Byte>() -> decodeSigned(source).toByte()
         typeOf<Int>() -> decodeSigned(source).toInt()
+        typeOf<Short>() -> decodeSigned(source).toShort()
         typeOf<Long>() -> decodeSigned(source).toLong()
         else ->
             throw IllegalArgumentException("can't decode to ${T::class.simpleName}")
@@ -75,9 +76,11 @@ inline fun <reified T : Any> IntCodec.encode(x: T, dout: DataSink) {
     when (x) {
         is UByte -> encodeUnsigned(x.toULong(), dout)
         is UInt -> encodeUnsigned(x.toULong(), dout)
+        is UShort -> encodeUnsigned(x.toULong(), dout)
         is ULong -> encodeUnsigned(x, dout)
         is Byte -> encodeSigned(x.toLong(), dout)
         is Int -> encodeSigned(x.toLong(), dout)
+        is Short -> encodeSigned(x.toLong(), dout)
         is Long -> encodeSigned(x, dout)
         else -> throw IllegalArgumentException("can't encode with varitn ${x::class.simpleName}: $x")
     }

src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt

@@ -11,6 +11,16 @@ fun longToBytes(value: Long): ByteArray {
     return result
 }
 
+fun intToBytes(value: Int): ByteArray {
+    var l = value
+    val result = ByteArray(8)
+    for (i in 3 downTo 0) {
+        result[i] = (l and 0xFF).toByte()
+        l = l shr 8
+    }
+    return result
+}
+
 /**
  * Convert 8 bytes to LE long
  */
@@ -23,6 +33,15 @@ fun bytesToLong(b: ByteArray): Long {
     return result
 }
 
+fun bytesToInt(b: ByteArray): Int {
+    var result: Int = 0
+    for (i in 0 until 4) {
+        result = result shl 8
+        result = result or (b[i].toInt() and 0xFF)
+    }
+    return result
+}
+
 
 private val hexDigits = "0123456789ABCDEF"
 

src/commonMain/kotlin/net.sergeych.bintools/smartint.kt

@@ -1,6 +1,6 @@
 @file:OptIn(ExperimentalUnsignedTypes::class)
 
-package com.icodici.ubdata
+package net.sergeych.bintools
 
 import net.sergeych.bintools.*
 

src/commonMain/kotlin/net.sergeych.bintools/varint.kt

@@ -1,8 +1,5 @@
-@file:OptIn(ExperimentalUnsignedTypes::class)
 
-package com.icodici.ubdata
+package net.sergeych.bintools
-
-import net.sergeych.bintools.*
 
 /**
  * Variable-length long integer encoding. the MSB (0x80) bit of each byte flags
@@ -14,15 +11,15 @@ import net.sergeych.bintools.*
  * same or even worse, see [Smartint] docs.
  */
 object Varint: IntCodec {
-    override fun encodeUnsigned(value: ULong, dout: DataSink) {
+    override fun encodeUnsigned(value: ULong, sink: DataSink) {
         var rest = value
         do {
             val x = (rest and 127u).toInt()
             rest = rest shr 7
             if (rest > 0u)
-                dout.writeByte(x or 0x80)
+                sink.writeByte(x or 0x80)
             else
-                dout.writeByte(x)
+                sink.writeByte(x)
 
         } while (rest > 0u)
     }

src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt

@@ -0,0 +1,68 @@
+package net.sergeych.bipack
+
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.AbstractDecoder
+import kotlinx.serialization.encoding.CompositeDecoder
+import kotlinx.serialization.modules.EmptySerializersModule
+import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.serializer
+import net.sergeych.bintools.DataSource
+import net.sergeych.bintools.readNumber
+import net.sergeych.bintools.toDataSource
+
+class BipackDecoder(val input: DataSource, var elementsCount: Int = 0) : AbstractDecoder() {
+    private var elementIndex = 0
+
+    override val serializersModule: SerializersModule = EmptySerializersModule
+    override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
+    override fun decodeByte(): Byte = input.readByte()
+    override fun decodeShort(): Short = input.readNumber()
+    override fun decodeInt(): Int = input.readNumber()
+    override fun decodeLong(): Long = input.readNumber()
+    override fun decodeFloat(): Float = input.readFloat()
+    override fun decodeDouble(): Double = input.readDouble()
+    override fun decodeChar(): Char = Char(input.readNumber<UInt>().toInt())
+
+    fun readBytes(): ByteArray {
+        val length = input.readNumber<UInt>()
+        return input.readBytes(length.toInt())
+    }
+
+    override fun decodeString(): String = readBytes().decodeToString()
+    override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readNumber<UInt>().toInt()
+
+    override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
+        if (elementIndex >= elementsCount) return CompositeDecoder.DECODE_DONE
+        return elementIndex++
+    }
+
+    override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder {
+        return BipackDecoder(
+            input,
+            if (descriptor.annotations.any { it is ExtendableFormat })
+                input.readNumber<UInt>().toInt()
+            else descriptor.elementsCount
+        )
+    }
+
+    override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
+        input.readNumber<UInt>().toInt().also {
+            elementsCount = it
+        }
+
+    override fun decodeNotNullMark(): Boolean = decodeBoolean()
+
+    @ExperimentalSerializationApi
+    override fun decodeNull(): Nothing? = null
+
+    companion object {
+        fun <T> decode(source: DataSource, deserializer: DeserializationStrategy<T>): T =
+            BipackDecoder(source).decodeSerializableValue(deserializer)
+
+        inline fun <reified T> decode(source: DataSource): T = decode(source, serializer())
+        inline fun <reified T> decode(source: ByteArray): T =
+            decode(source.toDataSource(), serializer())
+    }
+}

src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt

@@ -0,0 +1,66 @@
+package net.sergeych.bipack
+
+import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.AbstractEncoder
+import kotlinx.serialization.encoding.CompositeEncoder
+import kotlinx.serialization.modules.EmptySerializersModule
+import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.serializer
+import net.sergeych.bintools.ArrayDataSink
+import net.sergeych.bintools.DataSink
+import net.sergeych.bintools.writeI64
+import net.sergeych.bintools.writeNumber
+
+class BipackEncoder(val output: DataSink) : AbstractEncoder() {
+    override val serializersModule: SerializersModule = EmptySerializersModule
+    override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
+    override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
+    override fun encodeShort(value: Short) = output.writeNumber(value.toInt())
+    override fun encodeInt(value: Int) = output.writeNumber(value)
+
+    fun encodeUInt(value: UInt) = output.writeNumber(value)
+    override fun encodeLong(value: Long) = output.writeNumber(value)
+    override fun encodeFloat(value: Float) = output.writeNumber(value)
+    override fun encodeDouble(value: Double) = output.writeI64(value.toRawBits())
+    override fun encodeChar(value: Char) = output.writeNumber(value.code.toUInt())
+    override fun encodeString(value: String) {
+//        output.writeUTF(value)
+        writeBytes(value.encodeToByteArray())
+    }
+
+    fun writeBytes(value: ByteArray) {
+        output.writeNumber(value.size.toUInt())
+        output.writeBytes(value)
+    }
+
+    override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeNumber(index.toUInt())
+
+    override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
+        encodeUInt(collectionSize.toUInt())
+        return this
+    }
+
+    override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder {
+        if( descriptor.annotations.any { it is ExtendableFormat } )
+            encodeUInt(descriptor.elementsCount.toUInt())
+        return super.beginStructure(descriptor)
+    }
+
+    override fun encodeNull() = encodeBoolean(false)
+    override fun encodeNotNullMark() = encodeBoolean(true)
+
+    companion object {
+        fun <T> encode(serializer: SerializationStrategy<T>, value: T,sink: DataSink) {
+            val encoder = BipackEncoder(sink)
+            encoder.encodeSerializableValue(serializer, value)
+        }
+
+        fun <T> encode(serializer: SerializationStrategy<T>, value: T): ByteArray =
+            ArrayDataSink().also { encode(serializer, value, it)}.toByteArray()
+
+        inline fun <reified T> encode(value: T) = encode(serializer(), value)
+        inline fun <reified T> encode(value: T,sink: DataSink) = encode(serializer(), value, sink)
+
+    }
+}

src/commonMain/kotlin/net.sergeych.bipack/Extendable.kt

@@ -0,0 +1,18 @@
+package net.sergeych.bipack
+
+import kotlinx.serialization.SerialInfo
+
+/**
+ * If this annotation is presented in some @Serializable class defition, its instances
+ * will be serialized with leadinf number of fields. This allows to extend class later
+ * providing new parameters __to the end of the class__ and _with default values__.
+ *
+ * Whe deserializing such instances from previous version binaries, the new parameters
+ * will get default values.
+ *
+ * Serialized data of classes not market as ExtendableFormat could not be changed without
+ * breaking compatibility with existing serialized data.
+ */
+@Target(AnnotationTarget.CLASS)
+@SerialInfo
+annotation class ExtendableFormat

src/commonTest/kotlin/bintools/SmartintTest.kt

@@ -1,10 +1,6 @@
 package bintools
 
-import com.icodici.ubdata.Smartint
+import net.sergeych.bintools.*
-import com.icodici.ubdata.Varint
-import net.sergeych.bintools.decode
-import net.sergeych.bintools.encode
-import net.sergeych.bintools.encodeToHex
 import kotlin.test.Test
 import kotlin.test.assertEquals
 

src/commonTest/kotlin/bintools/VarintTest.kt

@@ -1,6 +1,6 @@
 package bintools
 
-import com.icodici.ubdata.Varint
+import net.sergeych.bintools.Varint
 import net.sergeych.bintools.decode
 import net.sergeych.bintools.encode
 import net.sergeych.bintools.encodeToHex

src/jvmTest/kotlin/net/sergeych/bipack/BipackEncoderTest.kt

@@ -0,0 +1,30 @@
+package net.sergeych.bipack
+
+import kotlinx.serialization.Serializable
+import net.sergeych.bintools.toDump
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+@Serializable
+@ExtendableFormat
+data class Foobar1(val bar: Int, val foo: Int = 117)
+
+@Serializable
+@ExtendableFormat
+data class Foobar2(val bar: Int, val foo: Int,val other: Int = -1)
+
+class BipackEncoderTest {
+
+    @Test
+    fun encodeSimple() {
+        val a = Foobar1(42)//, "bum")
+        println(BipackEncoder.encode(a).toDump())
+        val b = BipackDecoder.decode<Foobar1>(BipackEncoder.encode(a))
+        assertEquals(a, b)
+        val c = BipackDecoder.decode<Foobar2>(BipackEncoder.encode(a))
+        assertEquals(-1, c.other)
+//        assertEquals(a.foo, c.foo)
+        assertEquals(a.bar, c.bar)
+//        assertEquals(a.buzz, c.buzz)
+    }
+}