bipack: auto-detect unsigned inline types and add @Varint field codec

README.md

@@ -186,7 +186,17 @@ It adds four bytes to the serialized data.
 
 ## @Unsigned
 
-This __field annotation__ allows to store __integer fields__ of any size more compact by not saving the sign. It could be applied to both signed and unsigned integers of any size.
+This __field annotation__ allows storing signed integer fields (`Short`, `Int`, `Long`) in a more compact unsigned
+varint form when values are guaranteed non-negative.
+
+Unsigned Kotlin types (`UByte`, `UShort`, `UInt`, `ULong`) are detected automatically and do not need this annotation.
+
+## @Varint
+
+By default Bipack uses `Smartint` for variable-length integer coding. This __field annotation__ forces classic
+`Varint` codec for the annotated integer field.
+
+It can be combined with `@Unsigned` on signed integer fields.
 
 ## @FixedSize(size)
 
@@ -216,4 +226,3 @@ class Foo(
 // so:
 assertEquals("00 00 00 01 00 00 00 02", BipackEncoder.encode(Foo(0x100000002)).encodeToHex())
 ~~~
-

build.gradle.kts

@@ -62,7 +62,7 @@ kotlin {
             dependencies {
                 implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
                 // this is actually a bug: we need only the core, but bare core causes strange errors
-                implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0")
                 api("net.sergeych:mp_stools:[1.6.3,)")
                 implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.7.1")
             }

docs/bipack.md

@@ -17,7 +17,7 @@ Bipack is a common kotlinx serializer that works pretty much like any other `kot
 - [BipackEncoder] to serializes anything to bipack format.
 - [BipackDecoder] deserializes from bipack back.
 
-There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned] for integer data fields.
+There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned], [Varint] for integer data fields.
 
 # Package net.sergeych.bintools
 
@@ -29,4 +29,4 @@ In particular, see [Varint] and [Smartint] variable-length compact integer codec
 
 To write a code that compiles and runs, and most likely works on the
 JS, native, and JVM, we need some portable/compatible synchronization 
-primitives. This package is a collection of such.
+primitives. This package is a collection of such.

src/commonMain/kotlin/net.sergeych.bintools/varint.kt

@@ -2,8 +2,8 @@
 package net.sergeych.bintools
 
 /**
- * Variable-length long integer encoding. the MSB (0x80) bit of each byte flags
+ * Variable-length long integer encoding. The MSB (0x80) bit of each byte flags
- * that it is not the last one, and all ncecssary bits are encoded with 7-bit
+ * that it is not the last one, and all necessary bits are encoded with 7-bit
  * portions, LSB to MSB (big endian of sorts).
  *
  * There is slower but more compact encoding variant, [Smartint] that is better when

src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt

@@ -25,6 +25,7 @@ class BipackDecoder(
     private var elementIndex = 0
 
     private var nextIsUnsigned = false
+    private var nextIsVarint = false
     private var fixedSize = -1
     private var fixedNumber = false
 
@@ -33,6 +34,9 @@ class BipackDecoder(
     override fun decodeByte(): Byte = input.readByte()
     override fun decodeShort(): Short =
         if (fixedNumber) input.readI16()
+        else if (nextIsVarint)
+            if (nextIsUnsigned) input.readVarUInt().toShort()
+            else input.readVarInt().toShort()
         else if (nextIsUnsigned)
             input.readNumber<UInt>().toShort()
         else
@@ -40,10 +44,16 @@ class BipackDecoder(
 
     override fun decodeInt(): Int =
         if (fixedNumber) input.readI32()
+        else if (nextIsVarint)
+            if (nextIsUnsigned) input.readVarUInt().toInt()
+            else input.readVarInt()
         else if (nextIsUnsigned) input.readNumber<UInt>().toInt() else input.readNumber()
 
     override fun decodeLong(): Long =
         if (fixedNumber) input.readI64()
+        else if (nextIsVarint)
+            if (nextIsUnsigned) net.sergeych.bintools.Varint.decodeUnsigned(input).toLong()
+            else net.sergeych.bintools.Varint.decodeSigned(input)
         else if (nextIsUnsigned) input.readNumber<ULong>().toLong() else input.readNumber()
 
     override fun decodeFloat(): Float = input.readFloat()
@@ -63,9 +73,11 @@ class BipackDecoder(
         if (elementIndex >= elementsCount)
             return CompositeDecoder.DECODE_DONE
         nextIsUnsigned = false
+        nextIsVarint = false
         for (a in descriptor.getElementAnnotations(elementIndex)) {
             when (a) {
                 is Unsigned -> nextIsUnsigned = true
+                is Varint -> nextIsVarint = true
                 is FixedSize -> fixedSize = a.size
                 is Fixed -> fixedNumber = true
             }
@@ -73,6 +85,12 @@ class BipackDecoder(
         return elementIndex++
     }
 
+    override fun decodeInline(descriptor: SerialDescriptor): BipackDecoder {
+        if (descriptor.isUnsignedInlinePrimitive())
+            nextIsUnsigned = true
+        return this
+    }
+
     override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
         return if (deserializer == serializer<Instant>())
             Instant.fromEpochMilliseconds(decodeLong()) as T
@@ -136,6 +154,9 @@ class BipackDecoder(
     @ExperimentalSerializationApi
     override fun decodeNull(): Nothing? = null
 
+    private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
+        isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
+
     companion object {
         fun <T> decode(source: DataSource, deserializer: DeserializationStrategy<T>): T =
             BipackDecoder(source).decodeSerializableValue(deserializer)
@@ -156,4 +177,3 @@ class BipackDecoder(
 inline fun <reified T> ByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
 @Suppress("unused")
 inline fun <reified T> UByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
-

src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt

@@ -13,26 +13,38 @@ import kotlin.time.Instant
 class BipackEncoder(val output: DataSink) : AbstractEncoder() {
 
     private var nextIsUnsigned = false
+    private var nextIsVarint = false
     private var fixedSize: Int = -1
     private var fixedNumber: Boolean = false
 
     override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean =
         super.encodeElement(descriptor, index).also {
             nextIsUnsigned = false
+            nextIsVarint = false
             for (a in descriptor.getElementAnnotations(index)) {
                 when (a) {
                     is Unsigned -> nextIsUnsigned = true
+                    is Varint -> nextIsVarint = true
                     is FixedSize -> fixedSize = a.size
                     is Fixed -> fixedNumber = true
                 }
             }
         }
 
+    override fun encodeInline(descriptor: SerialDescriptor): BipackEncoder {
+        if (descriptor.isUnsignedInlinePrimitive())
+            nextIsUnsigned = true
+        return this
+    }
+
     override val serializersModule: SerializersModule = EmptySerializersModule()
     override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
     override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
     override fun encodeShort(value: Short) =
         if (fixedNumber) output.writeI16(value)
+        else if (nextIsVarint)
+            if (nextIsUnsigned) output.writeVarUInt(value.toUShort().toUInt())
+            else output.writeVarInt(value.toInt())
         else if (nextIsUnsigned)
             output.writeNumber(value.toUShort())
         else
@@ -41,6 +53,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
     override fun encodeInt(value: Int) =
         if (fixedNumber)
             output.writeI32(value)
+        else if (nextIsVarint)
+            if (nextIsUnsigned) output.writeVarUInt(value.toUInt())
+            else output.writeVarInt(value)
         else if (nextIsUnsigned) output.writeNumber(value.toUInt())
         else output.writeNumber(value)
 
@@ -48,6 +63,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
     override fun encodeLong(value: Long) =
         if (fixedNumber)
             output.writeI64(value)
+        else if (nextIsVarint)
+            if (nextIsUnsigned) net.sergeych.bintools.Varint.encodeUnsigned(value.toULong(), output)
+            else net.sergeych.bintools.Varint.encodeSigned(value, output)
         else if (nextIsUnsigned)
             output.writeNumber(value.toULong())
         else
@@ -111,6 +129,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
     override fun encodeNull() = encodeBoolean(false)
     override fun encodeNotNullMark() = encodeBoolean(true)
 
+    private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
+        isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
+
     companion object {
         fun <T> encode(serializer: SerializationStrategy<T>, value: T, sink: DataSink) {
             val encoder = BipackEncoder(sink)

src/commonMain/kotlin/net.sergeych.bipack/annotations.kt

@@ -43,14 +43,25 @@ annotation class Framed
 annotation class CrcProtected
 
 /**
- * Allow marking data fields as being serialized as unsigned (applicable also to signed fields lite Int, Long and
+ * Marks signed integer fields (`Int`, `Long`, `Short`) that should be encoded using unsigned variable-length format.
- * Short, if you are sure they will not be negative). As unsigned types are not cully supported by `kotlinx.serialization`
+ *
- * it is the only way to tell the serialized to use more compact unsigned variable length encoding.
+ * Native unsigned Kotlin types (`UInt`, `ULong`, `UShort`, `UByte`) are handled automatically by modern
+ * `kotlinx.serialization` and do not require this annotation.
  */
 @SerialInfo
 @Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
 annotation class Unsigned
 
+/**
+ * Marks integer fields that should use [net.sergeych.bintools.Varint] codec instead of default [net.sergeych.bintools.Smartint].
+ *
+ * Applicable to all integer types (`Byte`, `Short`, `Int`, `Long`, `UByte`, `UShort`, `UInt`, `ULong`).
+ * If used together with [Unsigned] on signed types, unsigned varint encoding is used.
+ */
+@SerialInfo
+@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
+annotation class Varint
+
 /**
  * Fixed size collection of a given size. __Use it only with collections!__
  *
@@ -102,5 +113,3 @@ class InvalidFrameHeaderException(reason: String = "Frame header does not match"
 class InvalidFrameCRCException : InvalidFrameException("Checksum CRC32 failed")
 
 class WrongCollectionSize(reason: String) : Exception(reason)
-
-

src/commonTest/kotlin/bipack/BipackEncoderTest.kt

@@ -2,6 +2,7 @@ package bipack
 
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import net.sergeych.bintools.Varint as VarintCodec
 import net.sergeych.bintools.encodeToHex
 import net.sergeych.bintools.toDump
 import net.sergeych.bipack.*
@@ -126,7 +127,6 @@ class BipackEncoderTest {
 
     @Serializable
     data class FBU(
-        @Unsigned
         val u: UInt,
         @Unsigned
         val i: Int,
@@ -377,11 +377,8 @@ class BipackEncoderTest {
     @Serializable
     data class VarUInts(
         val b: UByte,
-        @Unsigned
         val si: UShort,
-        @Unsigned
         val i: UInt,
-        @Unsigned
         val li: ULong,
     )
 
@@ -404,6 +401,33 @@ class BipackEncoderTest {
         println(yv)
     }
 
+    @Serializable
+    data class VI1(@Varint val i: Int)
+
+    @Serializable
+    data class VI2(@Varint val i: UInt)
+
+    @Serializable
+    data class VI3(@Varint @Unsigned val i: Int)
+
+    @Test
+    fun testVarintAnnotation() {
+        val v1 = VI1(1234567)
+        val p1 = BipackEncoder.encode(v1)
+        assertContentEquals(VarintCodec.encodeSigned(1234567L), p1)
+        assertEquals(v1, BipackDecoder.decode<VI1>(p1))
+
+        val v2 = VI2(0xF10203u)
+        val p2 = BipackEncoder.encode(v2)
+        assertContentEquals(VarintCodec.encodeUnsigned(0xF10203uL), p2)
+        assertEquals(v2, BipackDecoder.decode<VI2>(p2))
+
+        val v3 = VI3(1234567)
+        val p3 = BipackEncoder.encode(v3)
+        assertContentEquals(VarintCodec.encodeUnsigned(1234567uL), p3)
+        assertEquals(v3, BipackDecoder.decode<VI3>(p3))
+    }
+
     @Test
     fun testStrangeUnpack() {
         @Serializable
@@ -458,4 +482,4 @@ class BipackEncoderTest {
         t1(1)
         t1(-1)
     }
-}
+}