SergeychWorks/mp_bintools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

bipack: auto-detect unsigned inline types and add @Varint field codec

Browse Source
This commit is contained in:
Sergey Chernov 2026-03-25 02:49:52 +03:00
parent b8ac3e20e0
commit a9f07ddb6f
8 changed files with 101 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -186,7 +186,17 @@ It adds four bytes to the serialized data.
## @Unsigned
This __field annotation__ allows to store __integer fields__ of any size more compact by not saving the sign. It could be applied to both signed and unsigned integers of any size.
This __field annotation__ allows storing signed integer fields (`Short`, `Int`, `Long`) in a more compact unsigned
varint form when values are guaranteed non-negative.
Unsigned Kotlin types (`UByte`, `UShort`, `UInt`, `ULong`) are detected automatically and do not need this annotation.
## @Varint
By default Bipack uses `Smartint` for variable-length integer coding. This __field annotation__ forces classic
`Varint` codec for the annotated integer field.
It can be combined with `@Unsigned` on signed integer fields.
## @FixedSize(size)
@ -216,4 +226,3 @@ class Foo(
// so:
assertEquals("00 00 00 01 00 00 00 02", BipackEncoder.encode(Foo(0x100000002)).encodeToHex())
~~~

2
build.gradle.kts
View File

@ -62,7 +62,7 @@ kotlin {
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
// this is actually a bug: we need only the core, but bare core causes strange errors
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0")
api("net.sergeych:mp_stools:[1.6.3,)")
implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.7.1")
}

2
docs/bipack.md
View File

@ -17,7 +17,7 @@ Bipack is a common kotlinx serializer that works pretty much like any other `kot
- [BipackEncoder] to serializes anything to bipack format.
- [BipackDecoder] deserializes from bipack back.
There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned] for integer data fields.
There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned], [Varint] for integer data fields.
# Package net.sergeych.bintools

4
src/commonMain/kotlin/net.sergeych.bintools/varint.kt
View File

@ -2,8 +2,8 @@
package net.sergeych.bintools
/**
* Variable-length long integer encoding. the MSB (0x80) bit of each byte flags
* that it is not the last one, and all ncecssary bits are encoded with 7-bit
* Variable-length long integer encoding. The MSB (0x80) bit of each byte flags
* that it is not the last one, and all necessary bits are encoded with 7-bit
* portions, LSB to MSB (big endian of sorts).
*
* There is slower but more compact encoding variant, [Smartint] that is better when

22
src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt
View File

@ -25,6 +25,7 @@ class BipackDecoder(
private var elementIndex = 0
private var nextIsUnsigned = false
private var nextIsVarint = false
private var fixedSize = -1
private var fixedNumber = false
@ -33,6 +34,9 @@ class BipackDecoder(
override fun decodeByte(): Byte = input.readByte()
override fun decodeShort(): Short =
if (fixedNumber) input.readI16()
else if (nextIsVarint)
if (nextIsUnsigned) input.readVarUInt().toShort()
else input.readVarInt().toShort()
else if (nextIsUnsigned)
input.readNumber<UInt>().toShort()
else
@ -40,10 +44,16 @@ class BipackDecoder(
override fun decodeInt(): Int =
if (fixedNumber) input.readI32()
else if (nextIsVarint)
if (nextIsUnsigned) input.readVarUInt().toInt()
else input.readVarInt()
else if (nextIsUnsigned) input.readNumber<UInt>().toInt() else input.readNumber()
override fun decodeLong(): Long =
if (fixedNumber) input.readI64()
else if (nextIsVarint)
if (nextIsUnsigned) net.sergeych.bintools.Varint.decodeUnsigned(input).toLong()
else net.sergeych.bintools.Varint.decodeSigned(input)
else if (nextIsUnsigned) input.readNumber<ULong>().toLong() else input.readNumber()
override fun decodeFloat(): Float = input.readFloat()
@ -63,9 +73,11 @@ class BipackDecoder(
if (elementIndex >= elementsCount)
return CompositeDecoder.DECODE_DONE
nextIsUnsigned = false
nextIsVarint = false
for (a in descriptor.getElementAnnotations(elementIndex)) {
when (a) {
is Unsigned -> nextIsUnsigned = true
is Varint -> nextIsVarint = true
is FixedSize -> fixedSize = a.size
is Fixed -> fixedNumber = true
}
@ -73,6 +85,12 @@ class BipackDecoder(
return elementIndex++
}
override fun decodeInline(descriptor: SerialDescriptor): BipackDecoder {
if (descriptor.isUnsignedInlinePrimitive())
nextIsUnsigned = true
return this
}
override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
return if (deserializer == serializer<Instant>())
Instant.fromEpochMilliseconds(decodeLong()) as T
@ -136,6 +154,9 @@ class BipackDecoder(
@ExperimentalSerializationApi
override fun decodeNull(): Nothing? = null
private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
companion object {
fun <T> decode(source: DataSource, deserializer: DeserializationStrategy<T>): T =
BipackDecoder(source).decodeSerializableValue(deserializer)
@ -156,4 +177,3 @@ class BipackDecoder(
inline fun <reified T> ByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
@Suppress("unused")
inline fun <reified T> UByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)

21
src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt
View File

@ -13,26 +13,38 @@ import kotlin.time.Instant
class BipackEncoder(val output: DataSink) : AbstractEncoder() {
private var nextIsUnsigned = false
private var nextIsVarint = false
private var fixedSize: Int = -1
private var fixedNumber: Boolean = false
override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean =
super.encodeElement(descriptor, index).also {
nextIsUnsigned = false
nextIsVarint = false
for (a in descriptor.getElementAnnotations(index)) {
when (a) {
is Unsigned -> nextIsUnsigned = true
is Varint -> nextIsVarint = true
is FixedSize -> fixedSize = a.size
is Fixed -> fixedNumber = true
}
}
}
override fun encodeInline(descriptor: SerialDescriptor): BipackEncoder {
if (descriptor.isUnsignedInlinePrimitive())
nextIsUnsigned = true
return this
}
override val serializersModule: SerializersModule = EmptySerializersModule()
override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
override fun encodeShort(value: Short) =
if (fixedNumber) output.writeI16(value)
else if (nextIsVarint)
if (nextIsUnsigned) output.writeVarUInt(value.toUShort().toUInt())
else output.writeVarInt(value.toInt())
else if (nextIsUnsigned)
output.writeNumber(value.toUShort())
else
@ -41,6 +53,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeInt(value: Int) =
if (fixedNumber)
output.writeI32(value)
else if (nextIsVarint)
if (nextIsUnsigned) output.writeVarUInt(value.toUInt())
else output.writeVarInt(value)
else if (nextIsUnsigned) output.writeNumber(value.toUInt())
else output.writeNumber(value)
@ -48,6 +63,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeLong(value: Long) =
if (fixedNumber)
output.writeI64(value)
else if (nextIsVarint)
if (nextIsUnsigned) net.sergeych.bintools.Varint.encodeUnsigned(value.toULong(), output)
else net.sergeych.bintools.Varint.encodeSigned(value, output)
else if (nextIsUnsigned)
output.writeNumber(value.toULong())
else
@ -111,6 +129,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeNull() = encodeBoolean(false)
override fun encodeNotNullMark() = encodeBoolean(true)
private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
companion object {
fun <T> encode(serializer: SerializationStrategy<T>, value: T, sink: DataSink) {
val encoder = BipackEncoder(sink)

19
src/commonMain/kotlin/net.sergeych.bipack/annotations.kt
View File

@ -43,14 +43,25 @@ annotation class Framed
annotation class CrcProtected
/**
* Allow marking data fields as being serialized as unsigned (applicable also to signed fields lite Int, Long and
* Short, if you are sure they will not be negative). As unsigned types are not cully supported by `kotlinx.serialization`
* it is the only way to tell the serialized to use more compact unsigned variable length encoding.
* Marks signed integer fields (`Int`, `Long`, `Short`) that should be encoded using unsigned variable-length format.
*
* Native unsigned Kotlin types (`UInt`, `ULong`, `UShort`, `UByte`) are handled automatically by modern
* `kotlinx.serialization` and do not require this annotation.
*/
@SerialInfo
@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
annotation class Unsigned
/**
* Marks integer fields that should use [net.sergeych.bintools.Varint] codec instead of default [net.sergeych.bintools.Smartint].
*
* Applicable to all integer types (`Byte`, `Short`, `Int`, `Long`, `UByte`, `UShort`, `UInt`, `ULong`).
* If used together with [Unsigned] on signed types, unsigned varint encoding is used.
*/
@SerialInfo
@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
annotation class Varint
/**
* Fixed size collection of a given size. __Use it only with collections!__
*
@ -102,5 +113,3 @@ class InvalidFrameHeaderException(reason: String = "Frame header does not match"
class InvalidFrameCRCException : InvalidFrameException("Checksum CRC32 failed")
class WrongCollectionSize(reason: String) : Exception(reason)

32
src/commonTest/kotlin/bipack/BipackEncoderTest.kt
View File

@ -2,6 +2,7 @@ package bipack
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import net.sergeych.bintools.Varint as VarintCodec
import net.sergeych.bintools.encodeToHex
import net.sergeych.bintools.toDump
import net.sergeych.bipack.*
@ -126,7 +127,6 @@ class BipackEncoderTest {
@Serializable
data class FBU(
@Unsigned
val u: UInt,
@Unsigned
val i: Int,
@ -377,11 +377,8 @@ class BipackEncoderTest {
@Serializable
data class VarUInts(
val b: UByte,
@Unsigned
val si: UShort,
@Unsigned
val i: UInt,
@Unsigned
val li: ULong,
)
@ -404,6 +401,33 @@ class BipackEncoderTest {
println(yv)
}
@Serializable
data class VI1(@Varint val i: Int)
@Serializable
data class VI2(@Varint val i: UInt)
@Serializable
data class VI3(@Varint @Unsigned val i: Int)
@Test
fun testVarintAnnotation() {
val v1 = VI1(1234567)
val p1 = BipackEncoder.encode(v1)
assertContentEquals(VarintCodec.encodeSigned(1234567L), p1)
assertEquals(v1, BipackDecoder.decode<VI1>(p1))
val v2 = VI2(0xF10203u)
val p2 = BipackEncoder.encode(v2)
assertContentEquals(VarintCodec.encodeUnsigned(0xF10203uL), p2)
assertEquals(v2, BipackDecoder.decode<VI2>(p2))
val v3 = VI3(1234567)
val p3 = BipackEncoder.encode(v3)
assertContentEquals(VarintCodec.encodeUnsigned(1234567uL), p3)
assertEquals(v3, BipackDecoder.decode<VI3>(p3))
}
@Test
fun testStrangeUnpack() {
@Serializable