From a9f07ddb6f927170b9e8651a130299d296d2c53b Mon Sep 17 00:00:00 2001 From: sergeych Date: Wed, 25 Mar 2026 02:49:52 +0300 Subject: [PATCH] bipack: auto-detect unsigned inline types and add @Varint field codec --- README.md | 13 +++++-- build.gradle.kts | 2 +- docs/bipack.md | 4 +-- .../kotlin/net.sergeych.bintools/varint.kt | 4 +-- .../net.sergeych.bipack/BipackDecoder.kt | 22 +++++++++++- .../net.sergeych.bipack/BipackEncoder.kt | 21 ++++++++++++ .../kotlin/net.sergeych.bipack/annotations.kt | 19 ++++++++--- .../kotlin/bipack/BipackEncoderTest.kt | 34 ++++++++++++++++--- 8 files changed, 101 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 654570c..b677e44 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,17 @@ It adds four bytes to the serialized data. ## @Unsigned -This __field annotation__ allows to store __integer fields__ of any size more compact by not saving the sign. It could be applied to both signed and unsigned integers of any size. +This __field annotation__ allows storing signed integer fields (`Short`, `Int`, `Long`) in a more compact unsigned +varint form when values are guaranteed non-negative. + +Unsigned Kotlin types (`UByte`, `UShort`, `UInt`, `ULong`) are detected automatically and do not need this annotation. + +## @Varint + +By default Bipack uses `Smartint` for variable-length integer coding. This __field annotation__ forces classic +`Varint` codec for the annotated integer field. + +It can be combined with `@Unsigned` on signed integer fields. ## @FixedSize(size) @@ -216,4 +226,3 @@ class Foo( // so: assertEquals("00 00 00 01 00 00 00 02", BipackEncoder.encode(Foo(0x100000002)).encodeToHex()) ~~~ - diff --git a/build.gradle.kts b/build.gradle.kts index 613ea40..0ff3518 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ kotlin { dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") // this is actually a bug: we need only the core, but bare core causes strange errors - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0") api("net.sergeych:mp_stools:[1.6.3,)") implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.7.1") } diff --git a/docs/bipack.md b/docs/bipack.md index e0c2a43..ea406cf 100644 --- a/docs/bipack.md +++ b/docs/bipack.md @@ -17,7 +17,7 @@ Bipack is a common kotlinx serializer that works pretty much like any other `kot - [BipackEncoder] to serializes anything to bipack format. - [BipackDecoder] deserializes from bipack back. -There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned] for integer data fields. +There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned], [Varint] for integer data fields. # Package net.sergeych.bintools @@ -29,4 +29,4 @@ In particular, see [Varint] and [Smartint] variable-length compact integer codec To write a code that compiles and runs, and most likely works on the JS, native, and JVM, we need some portable/compatible synchronization -primitives. This package is a collection of such. \ No newline at end of file +primitives. This package is a collection of such. diff --git a/src/commonMain/kotlin/net.sergeych.bintools/varint.kt b/src/commonMain/kotlin/net.sergeych.bintools/varint.kt index fdddb3a..832615b 100644 --- a/src/commonMain/kotlin/net.sergeych.bintools/varint.kt +++ b/src/commonMain/kotlin/net.sergeych.bintools/varint.kt @@ -2,8 +2,8 @@ package net.sergeych.bintools /** - * Variable-length long integer encoding. the MSB (0x80) bit of each byte flags - * that it is not the last one, and all ncecssary bits are encoded with 7-bit + * Variable-length long integer encoding. The MSB (0x80) bit of each byte flags + * that it is not the last one, and all necessary bits are encoded with 7-bit * portions, LSB to MSB (big endian of sorts). * * There is slower but more compact encoding variant, [Smartint] that is better when diff --git a/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt b/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt index a0d8fed..869aca8 100644 --- a/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt +++ b/src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt @@ -25,6 +25,7 @@ class BipackDecoder( private var elementIndex = 0 private var nextIsUnsigned = false + private var nextIsVarint = false private var fixedSize = -1 private var fixedNumber = false @@ -33,6 +34,9 @@ class BipackDecoder( override fun decodeByte(): Byte = input.readByte() override fun decodeShort(): Short = if (fixedNumber) input.readI16() + else if (nextIsVarint) + if (nextIsUnsigned) input.readVarUInt().toShort() + else input.readVarInt().toShort() else if (nextIsUnsigned) input.readNumber().toShort() else @@ -40,10 +44,16 @@ class BipackDecoder( override fun decodeInt(): Int = if (fixedNumber) input.readI32() + else if (nextIsVarint) + if (nextIsUnsigned) input.readVarUInt().toInt() + else input.readVarInt() else if (nextIsUnsigned) input.readNumber().toInt() else input.readNumber() override fun decodeLong(): Long = if (fixedNumber) input.readI64() + else if (nextIsVarint) + if (nextIsUnsigned) net.sergeych.bintools.Varint.decodeUnsigned(input).toLong() + else net.sergeych.bintools.Varint.decodeSigned(input) else if (nextIsUnsigned) input.readNumber().toLong() else input.readNumber() override fun decodeFloat(): Float = input.readFloat() @@ -63,9 +73,11 @@ class BipackDecoder( if (elementIndex >= elementsCount) return CompositeDecoder.DECODE_DONE nextIsUnsigned = false + nextIsVarint = false for (a in descriptor.getElementAnnotations(elementIndex)) { when (a) { is Unsigned -> nextIsUnsigned = true + is Varint -> nextIsVarint = true is FixedSize -> fixedSize = a.size is Fixed -> fixedNumber = true } @@ -73,6 +85,12 @@ class BipackDecoder( return elementIndex++ } + override fun decodeInline(descriptor: SerialDescriptor): BipackDecoder { + if (descriptor.isUnsignedInlinePrimitive()) + nextIsUnsigned = true + return this + } + override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { return if (deserializer == serializer()) Instant.fromEpochMilliseconds(decodeLong()) as T @@ -136,6 +154,9 @@ class BipackDecoder( @ExperimentalSerializationApi override fun decodeNull(): Nothing? = null + private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean = + isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte") + companion object { fun decode(source: DataSource, deserializer: DeserializationStrategy): T = BipackDecoder(source).decodeSerializableValue(deserializer) @@ -156,4 +177,3 @@ class BipackDecoder( inline fun ByteArray.decodeFromBipack() = BipackDecoder.decode(this) @Suppress("unused") inline fun UByteArray.decodeFromBipack() = BipackDecoder.decode(this) - diff --git a/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt b/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt index 382e547..14875d2 100644 --- a/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt +++ b/src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt @@ -13,26 +13,38 @@ import kotlin.time.Instant class BipackEncoder(val output: DataSink) : AbstractEncoder() { private var nextIsUnsigned = false + private var nextIsVarint = false private var fixedSize: Int = -1 private var fixedNumber: Boolean = false override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean = super.encodeElement(descriptor, index).also { nextIsUnsigned = false + nextIsVarint = false for (a in descriptor.getElementAnnotations(index)) { when (a) { is Unsigned -> nextIsUnsigned = true + is Varint -> nextIsVarint = true is FixedSize -> fixedSize = a.size is Fixed -> fixedNumber = true } } } + override fun encodeInline(descriptor: SerialDescriptor): BipackEncoder { + if (descriptor.isUnsignedInlinePrimitive()) + nextIsUnsigned = true + return this + } + override val serializersModule: SerializersModule = EmptySerializersModule() override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0) override fun encodeByte(value: Byte) = output.writeByte(value.toInt()) override fun encodeShort(value: Short) = if (fixedNumber) output.writeI16(value) + else if (nextIsVarint) + if (nextIsUnsigned) output.writeVarUInt(value.toUShort().toUInt()) + else output.writeVarInt(value.toInt()) else if (nextIsUnsigned) output.writeNumber(value.toUShort()) else @@ -41,6 +53,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() { override fun encodeInt(value: Int) = if (fixedNumber) output.writeI32(value) + else if (nextIsVarint) + if (nextIsUnsigned) output.writeVarUInt(value.toUInt()) + else output.writeVarInt(value) else if (nextIsUnsigned) output.writeNumber(value.toUInt()) else output.writeNumber(value) @@ -48,6 +63,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() { override fun encodeLong(value: Long) = if (fixedNumber) output.writeI64(value) + else if (nextIsVarint) + if (nextIsUnsigned) net.sergeych.bintools.Varint.encodeUnsigned(value.toULong(), output) + else net.sergeych.bintools.Varint.encodeSigned(value, output) else if (nextIsUnsigned) output.writeNumber(value.toULong()) else @@ -111,6 +129,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() { override fun encodeNull() = encodeBoolean(false) override fun encodeNotNullMark() = encodeBoolean(true) + private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean = + isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte") + companion object { fun encode(serializer: SerializationStrategy, value: T, sink: DataSink) { val encoder = BipackEncoder(sink) diff --git a/src/commonMain/kotlin/net.sergeych.bipack/annotations.kt b/src/commonMain/kotlin/net.sergeych.bipack/annotations.kt index 8f4f911..0bd27bb 100644 --- a/src/commonMain/kotlin/net.sergeych.bipack/annotations.kt +++ b/src/commonMain/kotlin/net.sergeych.bipack/annotations.kt @@ -43,14 +43,25 @@ annotation class Framed annotation class CrcProtected /** - * Allow marking data fields as being serialized as unsigned (applicable also to signed fields lite Int, Long and - * Short, if you are sure they will not be negative). As unsigned types are not cully supported by `kotlinx.serialization` - * it is the only way to tell the serialized to use more compact unsigned variable length encoding. + * Marks signed integer fields (`Int`, `Long`, `Short`) that should be encoded using unsigned variable-length format. + * + * Native unsigned Kotlin types (`UInt`, `ULong`, `UShort`, `UByte`) are handled automatically by modern + * `kotlinx.serialization` and do not require this annotation. */ @SerialInfo @Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY) annotation class Unsigned +/** + * Marks integer fields that should use [net.sergeych.bintools.Varint] codec instead of default [net.sergeych.bintools.Smartint]. + * + * Applicable to all integer types (`Byte`, `Short`, `Int`, `Long`, `UByte`, `UShort`, `UInt`, `ULong`). + * If used together with [Unsigned] on signed types, unsigned varint encoding is used. + */ +@SerialInfo +@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY) +annotation class Varint + /** * Fixed size collection of a given size. __Use it only with collections!__ * @@ -102,5 +113,3 @@ class InvalidFrameHeaderException(reason: String = "Frame header does not match" class InvalidFrameCRCException : InvalidFrameException("Checksum CRC32 failed") class WrongCollectionSize(reason: String) : Exception(reason) - - diff --git a/src/commonTest/kotlin/bipack/BipackEncoderTest.kt b/src/commonTest/kotlin/bipack/BipackEncoderTest.kt index fb66d51..edf1932 100644 --- a/src/commonTest/kotlin/bipack/BipackEncoderTest.kt +++ b/src/commonTest/kotlin/bipack/BipackEncoderTest.kt @@ -2,6 +2,7 @@ package bipack import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import net.sergeych.bintools.Varint as VarintCodec import net.sergeych.bintools.encodeToHex import net.sergeych.bintools.toDump import net.sergeych.bipack.* @@ -126,7 +127,6 @@ class BipackEncoderTest { @Serializable data class FBU( - @Unsigned val u: UInt, @Unsigned val i: Int, @@ -377,11 +377,8 @@ class BipackEncoderTest { @Serializable data class VarUInts( val b: UByte, - @Unsigned val si: UShort, - @Unsigned val i: UInt, - @Unsigned val li: ULong, ) @@ -404,6 +401,33 @@ class BipackEncoderTest { println(yv) } + @Serializable + data class VI1(@Varint val i: Int) + + @Serializable + data class VI2(@Varint val i: UInt) + + @Serializable + data class VI3(@Varint @Unsigned val i: Int) + + @Test + fun testVarintAnnotation() { + val v1 = VI1(1234567) + val p1 = BipackEncoder.encode(v1) + assertContentEquals(VarintCodec.encodeSigned(1234567L), p1) + assertEquals(v1, BipackDecoder.decode(p1)) + + val v2 = VI2(0xF10203u) + val p2 = BipackEncoder.encode(v2) + assertContentEquals(VarintCodec.encodeUnsigned(0xF10203uL), p2) + assertEquals(v2, BipackDecoder.decode(p2)) + + val v3 = VI3(1234567) + val p3 = BipackEncoder.encode(v3) + assertContentEquals(VarintCodec.encodeUnsigned(1234567uL), p3) + assertEquals(v3, BipackDecoder.decode(p3)) + } + @Test fun testStrangeUnpack() { @Serializable @@ -458,4 +482,4 @@ class BipackEncoderTest { t1(1) t1(-1) } -} \ No newline at end of file +}