6 Commits
19 changed files with 1505 additions and 515 deletions

No files matched your search

+1
View File
@@ -7,3 +7,4 @@
.kotlin
/.gigaide/gigaide.properties
/java_pid40366.hprof
/local.properties
+15 -15
View File
@@ -4,22 +4,13 @@ Multiplatform binary tools collection, including portable serialization of the c
many useful tools to work with binary data, like CRC family checksums, dumps, etc. It works well also in the browser and
in native targets.
# Recent changes
# Important note
- 0.1.12 published on all platforms. many small additions.
Currently published version 0.4.0 for all platform is fully compatible with breaking kotlinx.datetime/kotlin.time migration as of Kotlin 2.2.21 and is __a recommended version to use__. Still there could be minor issues as:
- 0.1.11 added interesting collection classes: auto-sorted list with comparator, expirable cache, etc.
- version before `0.4.0` treated unsigned ints as signed ones, as older versions of `kotlinx.serialization` did. Since `0.4.0` UInt, ULong and UShort are treated as unsigned, no more @Unsigned annotation is needed.
- 0.1.7 built with kotlin 2.0.20 which contains important fix in wasmJS
- 0.1.6 add many useful features, added support to wasmJS and all other platforms. Note to wasmJS: it appears to be a bug in wasm compiler so BipackDecoder could cause wasm loading problem.
- 0.1.1: added serialized KVStorage with handy implementation on JVM and JS platforms and some required synchronization
tools.
-
- 0.1.0: uses modern kotlin 1.9.*, fixes problem with singleton or empty/object serialization
The last 1.8-based version is 0.0.8. Some fixes are not yet backported to it pls leave an issue of needed.
Most likely it will not cause any problems, but be informed.
# Documentation
@@ -197,7 +188,17 @@ It adds four bytes to the serialized data.
## @Unsigned
This __field annotation__ allows to store __integer fields__ of any size more compact by not saving the sign. It could be applied to both signed and unsigned integers of any size.
This __field annotation__ allows storing signed integer fields (`Short`, `Int`, `Long`) in a more compact unsigned
varint form when values are guaranteed non-negative.
Unsigned Kotlin types (`UByte`, `UShort`, `UInt`, `ULong`) are detected automatically and do not need this annotation.
## @Varint
By default Bipack uses `Smartint` for variable-length integer coding. This __field annotation__ forces classic
`Varint` codec for the annotated integer field.
It can be combined with `@Unsigned` on signed integer fields.
## @FixedSize(size)
@@ -227,4 +228,3 @@ class Foo(
// so:
assertEquals("00 00 00 01 00 00 00 02", BipackEncoder.encode(Foo(0x100000002)).encodeToHex())
~~~
+49 -17
View File
@@ -1,29 +1,33 @@
import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
plugins {
kotlin("multiplatform") version "2.2.20"
kotlin("plugin.serialization") version "2.2.20"
kotlin("multiplatform") version "2.3.20"
kotlin("plugin.serialization") version "2.2.21"
id("org.jetbrains.dokka") version "1.9.20"
id("com.android.library") version "8.7.2"
`maven-publish`
}
val serialization_version = "1.9.0"
group = "net.sergeych"
version = "0.2.2"
version = "0.4.0"
repositories {
google()
mavenCentral()
mavenLocal()
maven("https://maven.universablockchain.com/")
}
kotlin {
jvmToolchain(8)
jvmToolchain(17)
jvm()
androidTarget() {
// Ensure Android variant is published to Maven so consumers get androidMain APIs
publishLibraryVariants("release")
}
js {
browser()
nodejs()
browser()
nodejs()
}
macosArm64()
@@ -33,7 +37,6 @@ kotlin {
iosSimulatorArm64()
linuxX64()
linuxArm64()
mingwX64()
@OptIn(ExperimentalWasmDsl::class)
wasmJs {
@@ -53,15 +56,15 @@ kotlin {
languageSettings.optIn("kotlinx.serialization.ExperimentalSerializationApi")
languageSettings.optIn("kotlin.ExperimentalUnsignedTypes")
languageSettings.optIn("kotlin.contracts.ExperimentalContracts")
languageSettings.optIn("kotlin.time.ExperimentalTime")
}
val commonMain by getting {
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
// this is actually a bug: we need only the core, but bare core causes strange errors
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3")
api("net.sergeych:mp_stools:[1.6.1,)")
implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.5.0")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0")
api("net.sergeych:mp_stools:[1.6.3,)")
implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.7.1")
}
}
val nativeMain by creating {
@@ -85,19 +88,27 @@ kotlin {
}
val jvmMain by getting
val jvmTest by getting
val androidMain by getting {
dependencies {
// Android base64 and preferences are in the SDK; no extra deps required
}
}
val jsMain by getting {
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-browser:0.5.0")
}
}
val jsTest by getting
// val nativeTest by getting
val wasmJsMain by getting {
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-browser:0.5.0")
implementation("org.jetbrains.kotlinx:kotlinx-browser-wasm-js:0.5.0")
}
}
val wasmJsTest by getting {
dependencies {
// implementation("org.jetbrains.kotlinx:kotlinx-browser:0.3")
}
}
val wasmJsTest by getting
}
}
@@ -118,6 +129,27 @@ publishing {
}
}
}
android {
namespace = "net.sergeych.bintools"
compileSdk = 34
defaultConfig {
minSdk = 21
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
// Publish only release variant of the Android target so consumers (Android apps)
// resolve the platform artifact instead of common metadata, which is required
// to access Android-only APIs like net.sergeych.bintools.AndroidKV
publishing {
singleVariant("release") {
withSourcesJar()
}
}
}
tasks.dokkaHtml.configure {
outputDirectory.set(buildDir.resolve("dokka"))
dokkaSourceSets {
+1 -1
View File
@@ -17,7 +17,7 @@ Bipack is a common kotlinx serializer that works pretty much like any other `kot
- [BipackEncoder] to serializes anything to bipack format.
- [BipackDecoder] deserializes from bipack back.
There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned] for integer data fields.
There are also special annotation to fine tune the format: [Extendable], [Framed], [CrcProtected] for classes and [Unsigned], [Varint] for integer data fields.
# Package net.sergeych.bintools
+6 -1
View File
@@ -1,3 +1,8 @@
kotlin.code.style=official
kotlin.mpp.applyDefaultHierarchyTemplate=false
kotlin.daemon.jvmargs=-Xmx3072M
org.gradle.parallel=true
org.gradle.jvmargs=-Xmx4096M -Dfile.encoding=UTF-8
org.gradle.configuration-cache=true
org.gradle.caching=true
+1201 -457
View File
File diff suppressed because it is too large. Load diff
+18
View File
@@ -1,3 +1,21 @@
pluginManagement {
repositories {
google()
gradlePluginPortal()
mavenCentral()
mavenLocal()
}
}
dependencyResolutionManagement {
repositories {
google()
mavenCentral()
mavenLocal()
maven("https://maven.universablockchain.com/")
}
}
rootProject.name = "mp_bintools"
@@ -0,0 +1,65 @@
package net.sergeych.bintools
import android.content.Context
import android.content.SharedPreferences
import net.sergeych.bintools.AndroidKV.init
import net.sergeych.mp_tools.decodeBase64Compact
import net.sergeych.mp_tools.encodeToBase64Compact
/**
* Simple holder to provide Application [Context] for this library.
*
* Call [init] from your app (e.g., in `Application.onCreate`) before using
* [defaultNamedStorage].
*/
object AndroidKV {
@Volatile
private var appContext: Context? = null
fun init(context: Context) {
appContext = context.applicationContext
}
internal fun requireContext(): Context =
appContext ?: error("AndroidKV is not initialized. Call AndroidKV.init(applicationContext) first.")
}
/**
* Implementation of [KVStorage] backed by [SharedPreferences].
* Values are stored as base64-encoded strings to fit preferences capabilities.
*/
class SharedPreferencesKVStorage(
private val prefs: SharedPreferences,
keyPrefix: String = ""
) : KVStorage {
private val prefix = if (keyPrefix.isEmpty()) "" else "$keyPrefix:"
private fun k(key: String) = "$prefix$key"
override fun get(key: String): ByteArray? =
prefs.getString(k(key), null)?.decodeBase64Compact()
override fun set(key: String, value: ByteArray?) {
val e = prefs.edit()
val kk = k(key)
if (value == null) e.remove(kk)
else e.putString(kk, value.encodeToBase64Compact())
e.apply()
}
override val keys: Set<String>
get() {
val allKeys = prefs.all.keys
if (prefix.isEmpty()) return allKeys
return allKeys.filter { it.startsWith(prefix) }
.map { it.removePrefix(prefix) }
.toSet()
}
}
actual fun defaultNamedStorage(name: String): KVStorage {
val ctx = AndroidKV.requireContext()
val prefs = ctx.getSharedPreferences(name, Context.MODE_PRIVATE)
// We use separate preferences file per name, so prefix can be empty
return SharedPreferencesKVStorage(prefs)
}
@@ -0,0 +1,17 @@
package net.sergeych.synctools
import java.util.concurrent.locks.ReentrantLock
/**
* Android actual implementation mirrors JVM using [ReentrantLock].
*/
actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
private val access = ReentrantLock()
override fun lock() {
access.lock()
}
override fun unlock() {
access.unlock()
}
}
@@ -0,0 +1,22 @@
package net.sergeych.synctools
@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
actual class WaitHandle {
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
private val access = Object()
actual fun await(milliseconds: Long): Boolean {
return synchronized(access) {
try {
access.wait(milliseconds)
true
} catch (_: InterruptedException) {
false
}
}
}
actual fun wakeUp() {
synchronized(access) { access.notifyAll() }
}
}
@@ -218,7 +218,19 @@ class MemoryKVStorage(copyFrom: KVStorage? = null) : KVStorage {
* - otherwise, the folder will be created in "`~/.local_storage`" parent directory
* (which also will be created if needed).
*
* - For the native platorms it is not yet implemented (but will be soon).
* - In Android (new!) it is implemented using preferences, but initializing is needed in
* main activity `onCreate`, somewhat like:
* ```kotlin
* override fun onCreate(savedInstanceState: Bundle?) {
* super.onCreate(savedInstanceState)
* // set the Context which Preferences will be used for
* // defaultNamedStorage
* net.sergeych.bintools.AndroidKV.init(this)
* }
* ```
* see also `AndroidKV` and `SharedPreferencesKVStorage` for Android
*
* - For the native platforms it is not yet implemented (but will be soon).
*
* See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
* and `FileDataProvider` class on JVM target.
@@ -2,8 +2,8 @@
package net.sergeych.bintools
/**
* Variable-length long integer encoding. the MSB (0x80) bit of each byte flags
* that it is not the last one, and all ncecssary bits are encoded with 7-bit
* Variable-length long integer encoding. The MSB (0x80) bit of each byte flags
* that it is not the last one, and all necessary bits are encoded with 7-bit
* portions, LSB to MSB (big endian of sorts).
*
* There is slower but more compact encoding variant, [Smartint] that is better when
@@ -1,6 +1,5 @@
package net.sergeych.bipack
import kotlinx.datetime.Instant
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.KSerializer
@@ -12,6 +11,7 @@ import kotlinx.serialization.modules.EmptySerializersModule
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.serializer
import net.sergeych.bintools.*
import kotlin.time.Instant
/**
* Decode BiPack format. Note that it relies on [DataSource] so can throw [DataSource.EndOfData]
@@ -25,6 +25,7 @@ class BipackDecoder(
private var elementIndex = 0
private var nextIsUnsigned = false
private var nextIsVarint = false
private var fixedSize = -1
private var fixedNumber = false
@@ -33,6 +34,9 @@ class BipackDecoder(
override fun decodeByte(): Byte = input.readByte()
override fun decodeShort(): Short =
if (fixedNumber) input.readI16()
else if (nextIsVarint)
if (nextIsUnsigned) input.readVarUInt().toShort()
else input.readVarInt().toShort()
else if (nextIsUnsigned)
input.readNumber<UInt>().toShort()
else
@@ -40,10 +44,16 @@ class BipackDecoder(
override fun decodeInt(): Int =
if (fixedNumber) input.readI32()
else if (nextIsVarint)
if (nextIsUnsigned) input.readVarUInt().toInt()
else input.readVarInt()
else if (nextIsUnsigned) input.readNumber<UInt>().toInt() else input.readNumber()
override fun decodeLong(): Long =
if (fixedNumber) input.readI64()
else if (nextIsVarint)
if (nextIsUnsigned) net.sergeych.bintools.Varint.decodeUnsigned(input).toLong()
else net.sergeych.bintools.Varint.decodeSigned(input)
else if (nextIsUnsigned) input.readNumber<ULong>().toLong() else input.readNumber()
override fun decodeFloat(): Float = input.readFloat()
@@ -63,9 +73,11 @@ class BipackDecoder(
if (elementIndex >= elementsCount)
return CompositeDecoder.DECODE_DONE
nextIsUnsigned = false
nextIsVarint = false
for (a in descriptor.getElementAnnotations(elementIndex)) {
when (a) {
is Unsigned -> nextIsUnsigned = true
is Varint -> nextIsVarint = true
is FixedSize -> fixedSize = a.size
is Fixed -> fixedNumber = true
}
@@ -73,8 +85,14 @@ class BipackDecoder(
return elementIndex++
}
override fun decodeInline(descriptor: SerialDescriptor): BipackDecoder {
if (descriptor.isUnsignedInlinePrimitive())
nextIsUnsigned = true
return this
}
override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
return if (deserializer == Instant.serializer())
return if (deserializer == serializer<Instant>())
Instant.fromEpochMilliseconds(decodeLong()) as T
else
super.decodeSerializableValue(deserializer)
@@ -136,6 +154,9 @@ class BipackDecoder(
@ExperimentalSerializationApi
override fun decodeNull(): Nothing? = null
private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
companion object {
fun <T> decode(source: DataSource, deserializer: DeserializationStrategy<T>): T =
BipackDecoder(source).decodeSerializableValue(deserializer)
@@ -156,4 +177,3 @@ class BipackDecoder(
inline fun <reified T> ByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
@Suppress("unused")
inline fun <reified T> UByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
@@ -1,6 +1,5 @@
package net.sergeych.bipack
import kotlinx.datetime.Instant
import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encoding.AbstractEncoder
@@ -9,30 +8,43 @@ import kotlinx.serialization.modules.EmptySerializersModule
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.serializer
import net.sergeych.bintools.*
import kotlin.time.Instant
class BipackEncoder(val output: DataSink) : AbstractEncoder() {
private var nextIsUnsigned = false
private var nextIsVarint = false
private var fixedSize: Int = -1
private var fixedNumber: Boolean = false
override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean =
super.encodeElement(descriptor, index).also {
nextIsUnsigned = false
nextIsVarint = false
for (a in descriptor.getElementAnnotations(index)) {
when (a) {
is Unsigned -> nextIsUnsigned = true
is Varint -> nextIsVarint = true
is FixedSize -> fixedSize = a.size
is Fixed -> fixedNumber = true
}
}
}
override fun encodeInline(descriptor: SerialDescriptor): BipackEncoder {
if (descriptor.isUnsignedInlinePrimitive())
nextIsUnsigned = true
return this
}
override val serializersModule: SerializersModule = EmptySerializersModule()
override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
override fun encodeShort(value: Short) =
if (fixedNumber) output.writeI16(value)
else if (nextIsVarint)
if (nextIsUnsigned) output.writeVarUInt(value.toUShort().toUInt())
else output.writeVarInt(value.toInt())
else if (nextIsUnsigned)
output.writeNumber(value.toUShort())
else
@@ -41,6 +53,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeInt(value: Int) =
if (fixedNumber)
output.writeI32(value)
else if (nextIsVarint)
if (nextIsUnsigned) output.writeVarUInt(value.toUInt())
else output.writeVarInt(value)
else if (nextIsUnsigned) output.writeNumber(value.toUInt())
else output.writeNumber(value)
@@ -48,6 +63,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeLong(value: Long) =
if (fixedNumber)
output.writeI64(value)
else if (nextIsVarint)
if (nextIsUnsigned) net.sergeych.bintools.Varint.encodeUnsigned(value.toULong(), output)
else net.sergeych.bintools.Varint.encodeSigned(value, output)
else if (nextIsUnsigned)
output.writeNumber(value.toULong())
else
@@ -111,6 +129,9 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
override fun encodeNull() = encodeBoolean(false)
override fun encodeNotNullMark() = encodeBoolean(true)
private fun SerialDescriptor.isUnsignedInlinePrimitive(): Boolean =
isInline && serialName in setOf("kotlin.UInt", "kotlin.ULong", "kotlin.UShort", "kotlin.UByte")
companion object {
fun <T> encode(serializer: SerializationStrategy<T>, value: T, sink: DataSink) {
val encoder = BipackEncoder(sink)
@@ -43,14 +43,25 @@ annotation class Framed
annotation class CrcProtected
/**
* Allow marking data fields as being serialized as unsigned (applicable also to signed fields lite Int, Long and
* Short, if you are sure they will not be negative). As unsigned types are not cully supported by `kotlinx.serialization`
* it is the only way to tell the serialized to use more compact unsigned variable length encoding.
* Marks signed integer fields (`Int`, `Long`, `Short`) that should be encoded using unsigned variable-length format.
*
* Native unsigned Kotlin types (`UInt`, `ULong`, `UShort`, `UByte`) are handled automatically by modern
* `kotlinx.serialization` and do not require this annotation.
*/
@SerialInfo
@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
annotation class Unsigned
/**
* Marks integer fields that should use [net.sergeych.bintools.Varint] codec instead of default [net.sergeych.bintools.Smartint].
*
* Applicable to all integer types (`Byte`, `Short`, `Int`, `Long`, `UByte`, `UShort`, `UInt`, `ULong`).
* If used together with [Unsigned] on signed types, unsigned varint encoding is used.
*/
@SerialInfo
@Target(AnnotationTarget.FIELD, AnnotationTarget.PROPERTY)
annotation class Varint
/**
* Fixed size collection of a given size. __Use it only with collections!__
*
@@ -102,5 +113,3 @@ class InvalidFrameHeaderException(reason: String = "Frame header does not match"
class InvalidFrameCRCException : InvalidFrameException("Checksum CRC32 failed")
class WrongCollectionSize(reason: String) : Exception(reason)
@@ -2,11 +2,11 @@ package net.sergeych.collections
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import net.sergeych.mptools.withReentrantLock
import kotlin.time.Clock
import kotlin.time.Duration
import kotlin.time.Duration.Companion.seconds
import kotlin.time.Instant
/**
* MRU cache with expiration, with safe async concurrent access.
@@ -1,8 +1,8 @@
package net.sergecyh.diwan.tools
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import kotlin.time.Clock
import kotlin.time.Duration
import kotlin.time.Instant
/**
* Value with expiration.
@@ -1,8 +1,8 @@
package net.sergecyh.diwan.tools
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import kotlin.time.Clock
import kotlin.time.Duration
import kotlin.time.Instant
/**
* Experimental.
@@ -1,9 +1,8 @@
package bipack
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import net.sergeych.bintools.Varint as VarintCodec
import net.sergeych.bintools.encodeToHex
import net.sergeych.bintools.toDump
import net.sergeych.bipack.*
@@ -13,6 +12,8 @@ import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.time.Clock
import kotlin.time.Instant
@Serializable
data class Foobar1N(val bar: Int, val foo: Int = 117)
@@ -126,7 +127,6 @@ class BipackEncoderTest {
@Serializable
data class FBU(
@Unsigned
val u: UInt,
@Unsigned
val i: Int,
@@ -377,11 +377,8 @@ class BipackEncoderTest {
@Serializable
data class VarUInts(
val b: UByte,
@Unsigned
val si: UShort,
@Unsigned
val i: UInt,
@Unsigned
val li: ULong,
)
@@ -404,6 +401,33 @@ class BipackEncoderTest {
println(yv)
}
@Serializable
data class VI1(@Varint val i: Int)
@Serializable
data class VI2(@Varint val i: UInt)
@Serializable
data class VI3(@Varint @Unsigned val i: Int)
@Test
fun testVarintAnnotation() {
val v1 = VI1(1234567)
val p1 = BipackEncoder.encode(v1)
assertContentEquals(VarintCodec.encodeSigned(1234567L), p1)
assertEquals(v1, BipackDecoder.decode<VI1>(p1))
val v2 = VI2(0xF10203u)
val p2 = BipackEncoder.encode(v2)
assertContentEquals(VarintCodec.encodeUnsigned(0xF10203uL), p2)
assertEquals(v2, BipackDecoder.decode<VI2>(p2))
val v3 = VI3(1234567)
val p3 = BipackEncoder.encode(v3)
assertContentEquals(VarintCodec.encodeUnsigned(1234567uL), p3)
assertEquals(v3, BipackDecoder.decode<VI3>(p3))
}
@Test
fun testStrangeUnpack() {
@Serializable