+ByteChunk

.gitignore

@@ -4,3 +4,4 @@
 /gradle/wrapper/gradle-wrapper.jar
 /gradle/wrapper/gradle-wrapper.properties
 /node_modules
+.kotlin

README.md

@@ -1,6 +1,25 @@
 # Binary tools and BiPack serializer
 
-Multiplatform binary tools collection, including portable serialization of the compact and fast [Bipack] format, and many useful tools to work with binary data, like CRC family checksums, dumps, etc. It works well also in the browser and in native targets.
+Multiplatform binary tools collection, including portable serialization of the compact and fast [Bipack] format, and
+many useful tools to work with binary data, like CRC family checksums, dumps, etc. It works well also in the browser and
+in native targets.
+
+# Recent changes
+
+- 0.1.7 built with kotlin 2.0.20 which contains important fix in wasmJS
+
+- 0.1.6 add many useful features, added support to wasmJS and all other platforms. Note to wasmJS: it appears to be a bug in wasm compiler so BipackDecoder could cause wasm loading problem.
+
+- 0.1.1: added serialized KVStorage with handy implementation on JVM and JS platforms and some required synchronization
+  tools.
+-
+- 0.1.0: uses modern kotlin 1.9.*, fixes problem with singleton or empty/object serialization
+
+The last 1.8-based version is 0.0.8. Some fixes are not yet backported to it pls leave an issue of needed.
+
+# Documentation
+
+Aside of the samples in this readme please see [library documentation](https://code.sergeych.net/docs/mp_bintools/).
 
 # Usage
 
@@ -13,73 +32,132 @@ repositories {
 }
 ```
 
-And add dependecy to the proper place in yuor project like this:
+And add dependency to the proper place in your project like this:
 
 ```kotlin
 dependencies {
     // ...
-    implementation("net.sergeych:mp_bintools:0.0.3")
+    implementation("net.sergeych:mp_bintools:0.1.7")
 }
 ```
 
+## Calculating CRCs:
+
+~~~kotlin
+CRC.crc32("Hello".encodeToByteArray())
+CRC.crc16("happy".encodeToByteArray())
+CRC.crc8("world".encodeToByteArray())
+~~~
+
+## Binary effective serialization with Bipack:
+
+~~~kotlin
+@Serializable
+data class Foo(val bar: String,buzz: Int)
+
+val foo = Foo("bar", 42)
+val bytes = BipackEncoder.encode(foo)
+val bar: Foo = BipackDecoder.decode(bytes)
+assertEquals(foo, bar)
+~~~
+
+## Bipack-based auto-serializing storage:
+
+Allows easily storing whatever `@Serializable` data type using delegates
+and more:
+
+~~~kotlin
+val storage = defaultNamedStorage("test_mp_bintools")
+
+var foo by s1("unknown") // default value makes it a String
+foo = "bar"
+
+// nullable:
+var answer: Int? by storage.optStored()
+answer = 42
+
+s1.delete("foo")
+~~~
+
+## MotherPacker
+
+This conception allows switching encoding on the fly. Create some MotherPacker instance
+and pass it to your encoding/decoding code:
+
+~~~kotlin
+    @Serializable
+    data class FB1(val foo: Int,val bar: String)
+
+    // This is JSON implementation of MotherPacker:
+    val mp = JsonPacker()
+    // it packs and unpacks to JSON:
+    println(mp.pack(mapOf("foo" to 42)).decodeToString())
+    assertEquals("""{"foo":42}""", mp.pack(mapOf("foo" to 42)).decodeToString())
+    val x = mp.unpack<FB1>("""{"foo":42, "bar": "foo"}""".encodeToByteArray())
+    assertEquals(42, x.foo)
+    assertEquals("foo", x.bar)
+~~~
+
+There is also [MotherBipack] `MotherPacker` implementation using Bipack. You can add more formats
+easily by implementing `MotherPacker` interface.
+
 # Bipack
 
 ## Why?
 
 Bipack is a compact and efficient binary serialization library (and format) was designed with the following main goals:
 
-### Allow easy unpacking existing binary structures 
+### Allow easy unpacking existing binary structures
 
-Yuo describe your structure as `@Serializable` classes, and - voila, bipack decodes and encodes it for you! We aim to make it really easy to convert data from other binary formats by adding more format annotations
+Yuo describe your structure as `@Serializable` classes, and - voilà, bipack decodes and encodes it for you! We aim to make it really easy to convert data from other binary formats by adding more format annotations
 
 ### Be as compact as possible
 
-For this reason it is a binary notation, it uses binary form for decimal numbers and can use variery of encoding for
+For this reason it is a binary notation, it uses binary form for decimal numbers and can use a variety of encoding for
 integers:
 
 #### Varint
 
-Variable-length compact encoding is used internally in some cases. It uses a 0x80 bit in every byte to mark coninuation.
+Variable-length compact encoding is used internally in some cases. It uses a 0x80 bit in every byte to mark continuation.
 See `object Varint`.
 
 #### Smartint
 
-Variable-length compact encoding for signed and unsigned integers use as few bytes as possible to encode integers. It is
+Variable-length compact encoding for signed and unsigned integers uses as few bytes as possible to encode integers. It is used automatically when serializing integers. It is slightly more sophisticated than straight `Varint`.
-used automatically when serializing integers. It is slightly more sophisticated than straight `Varint`.
 
 ### Do not reveal information about stored data
 
-Many extendable formats, like JSON, BSON, BOSS and may others are keeping data in key-value pairs. While it is good in
+Many extendable formats, like JSON, BSON, BOSS and may others are keeping data in key-value pairs. While it is good in many aspects, it has some disadvantages: it uses more space, and it reveals inner data structure to the world. It is possible to unpack such formats with zero information about inner structure.
-many aspets, it has a clear disadvantages: it uses more space, and it reveals inner data structure to the world. It is
-possible to unpack such formats with zero information about inner structure.
 
-Bipack does not store field names, so it is not possible to unpack or interpret it without knowledge of the data
+Bipack does not store field names, so it is not possible to unpack or interpret it without knowledge of the data structure. Only probabilistic analysis. Let's not make the life of attacker easier :)
-structure. Only probablistic analysis. Let's not make life of attacker easier :)
 
-### - allow upgrading data structures with backward compatibility
+### -- allows upgrading data structures with backward compatibility
 
-The dark side of serialization formats of this kind is that you can't change the structures without either loosing
+The serialization formats of this kind have a dark side: you can't change the structures without either losing backward compatibility with already serialized data or using voluminous boilerplate code to implement some sort of versioning.
-backward compatibility with already serialzied data or using volumous boilerplate code to implement some sort of
-versioning.
 
-Not to waste space and reveal more information that needed Bipack allows extending classes marked as [@Extendable] to be
+Not to waste space
-extended with more data _appended to the end of list of fields with required defaul values_. For such classes, Bipack stores the number of actually serialized fields and atuomatically uses default values for non-serialized ones when unpacking
+and reveal more information that needed Bipack allows
-old data.
+extending classes marked
+as [@Extendable] to be extended with more data _appended to the end of the field list with required default values_.
+For such classes,
+Bipack stores the number of actually serialized fields
+and automatically uses default values for non-serialized ones when unpacking old data.
 
 ### Protect data with framing and CRC
 
-When needed, serialization lobrary allow to store/check CRC32 tag of the structure name with `@Framed` (can be overriden
+When needed,
-as usual with `@SerialName`), or be followed with CRC32 of the serialized binary data, that will be checked on
+a serialization library allow to store/check CRC32 tag of the structure name with
-deserialization, using `@CrcProtected`. This allows checking the data consistency out of the box and only where needed.
+`@Framed` (can be overridden as usual with `@SerialName`), or be followed with CRC32 of the serialized binary data, that will be checked on deserialization, using `@CrcProtected`. This allows checking the data consistency out of the box and only where needed.
 
 # Usage
 
-Use kotlinx serializatino as usual. There are the following Bipack-specific annotations at your disposal (can be combined):
+Use `kotlinx.serialization` as usual. There are the following Bipack-specific annotations at your disposal (can be
+combined):
 
 ## @Extendable
 
-Classes marked this way store number of fields. It allows to add to the class data more fields, to the end of list, with
+Classes marked this way store number of fields. It allows adding to the class data more fields, to the end of the list, with
-default initializers, keeping backward compatibility. For example if you have serialized:
+default initializers, keeping backward compatibility. For example, if you have serialized:
 
 ```kotlin
 @Serializable
@@ -95,28 +173,27 @@ and then decided to add a field:
 data class foo(val i: Int, val bar: String = "buzz")
 ```
 
-It adds 1 or more bytes to the serialized data (field counts in `Varint` format)
+It adds one or more bytes to the serialized data (field counts in `Varint` format)
 
-Bipack will properly deserialize the data serialzied for an old version.
+Bipack will properly deserialize the data serialized for an old version.
 
 ## @CrcProtected
 
 Bipack will calculate and store CRC32 of serialized data at the end, and automatically check it on deserializing
 throwing `InvalidFrameCRCException` if it does not match.
 
-It adds 4 bytes to the serialized data.
+It adds four bytes to the serialized data.
 
 ## @Framed
 
 Put the CRC32 of the serializing class name (`@SerialName` allows to change it as usual) and checks it on deserializing.
 Throws `InvalidFrameHeaderException` if it does not match.
 
-It adds 4 bytes to the serialized data.
+It adds four bytes to the serialized data.
 
-## @Unisgned
+## @Unsigned
 
-This __field annontation__ allows to store __integer fields__ of any size more compact by not saving the sign. Could be
+This __field annotation__ allows to store __integer fields__ of any size more compact by not saving the sign. It could be applied to both signed and unsigned integers of any size.
-applyed to both signed and unsigned integers of any size.
 
 ## @FixedSize(size)
 
@@ -125,7 +202,7 @@ at least one byte.
 
 ## @Fixed
 
-Can be used with any integer type to store/restor it as is, fixed-size, big-endian:
+Can be used with any integer type to store/restore it as is, fixed-size, big-endian:
 
 - Short, UShort: 2 bytes
 - Int, UInt: 4 bytes

bin/pubdocs

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+./gradlew dokkaHtml
+rsync -avz ./build/dokka/* code.sergeych.net:/bigstore/sergeych_pub/code/docs/mp_bintools

build.gradle.kts

@@ -1,14 +1,14 @@
 plugins {
-    kotlin("multiplatform") version "1.8.20"
+    kotlin("multiplatform") version "2.0.20"
-    kotlin("plugin.serialization") version "1.8.20"
+    kotlin("plugin.serialization") version "2.0.20"
-    id("org.jetbrains.dokka") version "1.6.0"
+    id("org.jetbrains.dokka") version "1.9.20"
     `maven-publish`
 }
 
-val serialization_version = "1.3.4"
+val serialization_version = "1.6.5-SNAPSHOT"
 
 group = "net.sergeych"
-version = "0.0.4"
+version = "0.1.8-SNAPSHOT"
 
 repositories {
     mavenCentral()
@@ -17,72 +17,79 @@ repositories {
 }
 
 kotlin {
-    jvm {
+    jvmToolchain(8)
-        compilations.all {
+    jvm()
-            kotlinOptions.jvmTarget = "1.8"
+    js {
-        }
+      browser()
-        withJava()
+      nodejs()
-        testRuns["test"].executionTask.configure {
-            useJUnitPlatform()
-        }
     }
-    js(IR) {
-        browser {
-            testTask {
-                useKarma {
-                    // /home/sergeych/snap/firefox/common/.mozilla/firefox/iff469o9.default
-                    //  /home/sergeych/snap/firefox/common/.mozilla/firefox/iff469o9.default
-//                    useFirefox()
-                    useChromeHeadless()
-//                    useSafari()
-                }
-            }
-//            commonWebpackConfig {
-//                cssSupport.enabled = true
-//            }
-        }
-        nodejs {
-            testTask {
 
-            }
+    macosArm64()
+    iosX64()
+    iosArm64()
+    macosX64()
+    iosSimulatorArm64()
+    linuxX64()
+    linuxArm64()
+    mingwX64()
+
+    wasmJs {
+        browser()
+        binaries.executable()
+    }
+
+
+    mingwX64() {
+        binaries.staticLib {
+            baseName = "mp_bintools"
         }
     }
 
-    val hostOs = System.getProperty("os.name")
-    val isMingwX64 = hostOs.startsWith("Windows")
-    val nativeTarget = when {
-        hostOs == "Mac OS X" -> macosX64("native")
-        hostOs == "Linux" -> linuxX64("native")
-        isMingwX64 -> mingwX64("native")
-        else -> throw GradleException("Host OS is not supported in Kotlin/Native.")
-    }
-
-    
     sourceSets {
         all {
             languageSettings.optIn("kotlinx.serialization.ExperimentalSerializationApi")
+            languageSettings.optIn("kotlin.ExperimentalUnsignedTypes")
+            languageSettings.optIn("kotlin.contracts.ExperimentalContracts")
         }
         val commonMain by getting {
             dependencies {
-                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.1")
                 // this is actually a bug: we need only the core, but bare core causes strange errors
-                implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.0")
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3")
-//                api("net.sergeych:mp_stools:[1.3.3,)")
+                api("net.sergeych:mp_stools:[1.4.7,)")
-                implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.4.0")
+                implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.5.0")
             }
         }
+        val nativeMain by creating {
+            dependsOn(commonMain)
+            dependencies {
+            }
+        }
+        val linuxX64Main by getting {
+            dependsOn(nativeMain)
+        }
+        val linuxArm64Main by getting {
+            dependsOn(nativeMain)
+        }
+
         val commonTest by getting {
             dependencies {
                 implementation(kotlin("test"))
-                implementation("net.sergeych:mp_stools:1.4.1")
             }
         }
         val jvmMain by getting
         val jvmTest by getting
-        val jsMain by getting
+        val jsMain by getting {
+            dependencies {
+            }
+        }
         val jsTest by getting
-        val nativeMain by getting
+//        val nativeTest by getting
-        val nativeTest by getting
+        val wasmJsMain by getting {
+            dependencies {
+            }
+        }
+        val wasmJsTest by getting
     }
 
     publishing {

docs/bipack.md

@@ -1,6 +1,13 @@
 # Module mp_bintools
 
-This library contains a `Bipack` binary format serializer, see [net.sergeych.bipack.BipackEncoder] and [net.sergeych.bipack.BipackDecoder]. Also, there are many general-purpose utilities that simplify binary data processing, see package [net.sergeych.bintools] below. 
+## Collection of binary tools
+
+Most interesting:
+
+- Full set of CRC: [net.sergeych.bintools.CRC]
+- Binary bit-effective __BiPack format__ serializer: [net.sergeych.bipack.BipackEncoder] and [net.sergeych.bipack.BipackDecoder]. Also typed key-value storage for it, see [net.sergeych.bipack.KVStorage] and its delegates and [net.sergeych.bipack.defaultNamedStorage].
+- Multiplatform synchronization tools, that works the same and properly on JS, native and JVM, see [net.sergeych.synctools]
+- many general-purpose utilities that simplify binary data processing, see package [net.sergeych.bintools] below. 
 
 
 # Package net.sergeych.bipack
@@ -16,4 +23,10 @@ There are also special annotation to fine tune the format: [Extendable], [Framed
 
 General-purpose binary tools: encoding to bytes, hex, binary dumps. variable length integer, ect. Most of it is used internally by bipack serializers, see [net.sergeych.bipack] for details.
 
-In particular, see [Varint] and [Smartint] variable-length compact integer codecs and also [DataSource] and [DataSink] multiplatform synchronous read/write interfaces.
+In particular, see [Varint] and [Smartint] variable-length compact integer codecs and also [DataSource] and [DataSink] multiplatform synchronous read/write interfaces.
+
+# Package net.sergeych.synctools
+
+To write a code that compiles and runs, and most likely works on the
+JS, native, and JVM, we need some portable/compatible synchronization 
+primitives. This package is a collection of such.

gradle.properties

@@ -1,2 +1,3 @@
 kotlin.code.style=official
 kotlin.js.compiler=ir
+kotlin.mpp.applyDefaultHierarchyTemplate=false

gradlew

@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,13 +80,11 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# This is normally unused
-
+# shellcheck disable=SC2034
-APP_NAME="Gradle"
 APP_BASE_NAME=${0##*/}
-
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,22 +131,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -193,11 +198,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
+
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-#     shell script including quotes and variable substitutions, so put them in
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-#     double quotes to make sure that they get re-expanded; and
+
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
@@ -205,6 +214,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,8 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+set EXIT_CODE=%ERRORLEVEL%
-exit /b 1
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

src/commonMain/kotlin/net.sergeych.bintools/ByteChunk.kt

@@ -0,0 +1,80 @@
+package net.sergeych.bintools
+
+import kotlinx.serialization.Serializable
+import kotlin.math.min
+import kotlin.random.Random
+
+/**
+ * Bytes sequence with comparison, concatenation, and string representation,
+ * could be used as hash keys for pure binary values, etc.
+ */
+@Suppress("unused")
+@Serializable
+class ByteChunk(val data: UByteArray): Comparable<ByteChunk> {
+
+    val size: Int get() = data.size
+
+    /**
+     * Per-byte comparison also of different length. From two chunks
+     * of different size but equal beginning, the shorter is considered
+     * the smaller.
+     */
+    override fun compareTo(other: ByteChunk): Int {
+        val limit = min(size, other.size)
+        for( i in 0 ..< limit) {
+            val own = data[i]
+            val their = other.data[i]
+            if( own < their) return -1
+            else if( own > their) return 1
+        }
+        if( size < other.size ) return -1
+        if( size > other.size ) return 1
+        return 0
+    }
+
+    /**
+     * Equal chunks means content equality.
+     */
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ByteChunk) return false
+
+        return data contentEquals other.data
+    }
+
+    /**
+     * Content-based hash code
+     */
+    override fun hashCode(): Int {
+        return data.contentHashCode()
+    }
+
+    /**
+     * hex representation of data
+     */
+    override fun toString(): String = hex
+
+    /**
+     * Hex encoded data
+     */
+    val hex by lazy { data.encodeToHex() }
+
+    /**
+     * human-readable dump
+     */
+    val dump by lazy { data.toDump() }
+
+    /**
+     * Concatenate two chunks and return new one
+     */
+    operator fun plus(other: ByteChunk): ByteChunk = ByteChunk(data + other.data)
+
+    companion object {
+        fun fromHex(hex: String): ByteChunk = ByteChunk(hex.decodeHex().asUByteArray())
+        fun random(size: Int=16): ByteChunk = Random.nextBytes(size).asChunk()
+    }
+}
+
+fun ByteArray.asChunk() = ByteChunk(this.asUByteArray())
+@Suppress("unused")
+fun UByteArray.asChunk() = ByteChunk(this)

src/commonMain/kotlin/net.sergeych.bintools/CRC.kt

@@ -27,6 +27,9 @@ interface CRC<T> {
         fun crc8(data: ByteArray, polynomial: UByte = 0xA7.toUByte()): UByte =
             CRC8(polynomial).also { it.update(data) }.value
 
+        fun crc8(data: UByteArray, polynomial: UByte = 0xA7.toUByte()): UByte =
+            CRC8(polynomial).also { it.update(data) }.value
+
         /**
          * Calculate CRC16 for a data array using a given polynomial (CRC16-CCITT polynomial (0x1021) by default)
          */
@@ -45,6 +48,7 @@ infix fun UShort.shl(bitCount: Int): UShort = (this.toUInt() shl bitCount).toUSh
 infix fun UShort.shr(bitCount: Int): UShort = (this.toUInt() shr bitCount).toUShort()
 
 infix fun UByte.shl(bitCount: Int): UByte = (this.toUInt() shl bitCount).toUByte()
+@Suppress("unused")
 infix fun UByte.shr(bitCount: Int): UByte = (this.toUInt() shr bitCount).toUByte()
 
 fun UByte.toBigEndianUShort(): UShort = this.toUShort() shl 8

src/commonMain/kotlin/net.sergeych.bintools/DataKVStorage.kt

@@ -0,0 +1,135 @@
+package net.sergeych.bintools
+
+import net.sergeych.bipack.BipackDecoder
+import net.sergeych.bipack.BipackEncoder
+import net.sergeych.mp_logger.LogTag
+import net.sergeych.mp_logger.Loggable
+import net.sergeych.mp_logger.debug
+import net.sergeych.synctools.AtomicCounter
+import net.sergeych.synctools.ProtectedOp
+import net.sergeych.synctools.WaitHandle
+import net.sergeych.synctools.withLock
+
+private val ac = AtomicCounter(0)
+
+class DataKVStorage(private val provider: DataProvider) : KVStorage,
+    Loggable by LogTag("DKVS${ac.incrementAndGet()}") {
+
+    data class Lock(val name: String) {
+        private val exclusive = ProtectedOp()
+
+        private var readerCount = 0
+
+        private var pulses = WaitHandle()
+
+        fun <T> lockExclusive(f: () -> T): T {
+            while (true) {
+                exclusive.withLock {
+                    if (readerCount == 0) {
+                        return f()
+                    }
+                }
+                pulses.await()
+            }
+        }
+
+        fun <T> lockRead(f: () -> T): T {
+            try {
+                exclusive.withLock { readerCount++ }
+                return f()
+            } finally {
+                exclusive.withLock { readerCount-- }
+                if (readerCount == 0) pulses.wakeUp()
+            }
+        }
+    }
+
+
+    private val locks = mutableMapOf<String, Lock>()
+    private val access = ProtectedOp()
+    private val keyIds = mutableMapOf<String, Int>()
+    private var lastId: Int = 0
+
+    init {
+        access.withLock {
+            // TODO: read keys
+            for (fn in provider.list()) {
+                debug { "Scanning: $fn" }
+                if (fn.endsWith(".d")) {
+                    val id = fn.dropLast(2).toInt(16)
+                    debug  { "found data record: $fn -> $id" }
+                    val name = provider.read(fn) { BipackDecoder.decode<String>(it) }
+                    keyIds[name] = id
+                    if (id > lastId) lastId = id
+                } else debug { "ignoring record $fn" }
+            }
+        }
+    }
+
+
+    /**
+     * Important: __it must be called with locked access op!__
+     */
+    private fun lockFor(name: String): Lock = locks.getOrPut(name) { Lock(name) }
+
+    private fun recordName(id: Int) = "${id.toString(16)}.d"
+    private fun <T> read(name: String, f: (DataSource) -> T): T {
+        val lock: Lock
+        val id: Int
+        // global lock: fast
+        access.withLock {
+            id = keyIds[name] ?: throw DataProvider.NotFoundException()
+            lock = lockFor(name)
+        }
+        // per-name lock: slow
+        return lock.lockRead {
+            provider.read(recordName(id), f)
+        }
+    }
+
+    private fun write(name: String, f: (DataSink) -> Unit) {
+        val lock: Lock
+        val id: Int
+        // global lock: fast
+        access.withLock {
+            id = keyIds[name] ?: (++lastId).also { keyIds[name] = it }
+            lock = lockFor(name)
+        }
+        // per-name lock: slow
+        lock.lockExclusive { provider.write(recordName(id), f) }
+    }
+
+    private fun deleteEntry(name: String) {
+        // fast pre-check:
+        if (name !in keyIds) return
+        // global lock: we can't now detect concurrent delete + write ops, so exclusive:
+        access.withLock {
+            val id = keyIds[name] ?: return
+            provider.delete(recordName(id))
+            locks.remove(name)
+            keyIds.remove(name)
+        }
+    }
+
+    override fun get(key: String): ByteArray? = try {
+        read(key) {
+            BipackDecoder.decode<String>(it)
+            // notice: not nullable byte array here!
+            BipackDecoder.decode<ByteArray>(it)
+        }
+    } catch (_: DataProvider.NotFoundException) {
+        null
+    }
+
+    override fun set(key: String, value: ByteArray?) {
+        if (value == null) {
+            deleteEntry(key)
+        } else write(key) {
+            BipackEncoder.encode(key, it)
+            BipackEncoder.encode(value, it)
+        }
+    }
+
+    override val keys: Set<String>
+        get() = access.withLock { keyIds.keys }
+}

src/commonMain/kotlin/net.sergeych.bintools/DataProvider.kt

@@ -0,0 +1,33 @@
+package net.sergeych.bintools
+
+/**
+ * Abstraction of some file- or named_record- storage. It could be a filesystem
+ * on native and JVM targets and indexed DB or session storage in the browser.
+ */
+interface DataProvider {
+    open class Error(msg: String,cause: Throwable?=null): Exception(msg,cause)
+    class NotFoundException(msg: String="record not found",cause: Throwable? = null): Error(msg, cause)
+    class WriteFailedException(msg: String="can't write", cause: Throwable?=null): Error(msg,cause)
+
+    /**
+     * Read named record/file
+     * @throws NotFoundException
+     */
+    fun <T>read(name: String,f: (DataSource)->T): T
+
+    /**
+     * Write to a named record / file.
+     * @throws WriteFailedException
+     */
+    fun write(name: String,f: (DataSink)->Unit)
+
+    /**
+     * Delete if exists, or do nothing.
+     */
+    fun delete(name: String)
+
+    /**
+     * List all record names in this source
+     */
+    fun list(): List<String>
+}

src/commonMain/kotlin/net.sergeych.bintools/DataSink.kt

@@ -1,5 +1,6 @@
 package net.sergeych.bintools
 
+@Suppress("unused")
 interface DataSink {
 
     fun writeByte(data: Byte)
@@ -24,9 +25,9 @@ interface DataSink {
     }
 
     fun writeVarUInt(value: UInt) { Varint.encodeUnsigned(value.toULong(), this)}
-    fun writeVarInt(value: UInt) { Varint.encodeSigned(value.toLong(), this)}
+    fun writeVarInt(value: Int) { Varint.encodeSigned(value.toLong(), this)}
     fun writeSmartUInt(value: UInt) { Smartint.encodeUnsigned(value.toULong(), this)}
-    fun writeSmartInt(value: UInt) { Smartint.encodeSigned(value.toLong(), this)}
+    fun writeSmartInt(value: Int) { Smartint.encodeSigned(value.toLong(), this)}
 }
 
 inline fun <reified T:Any>DataSink.writeNumber(value: T) {

src/commonMain/kotlin/net.sergeych.bintools/DataSource.kt

@@ -6,6 +6,7 @@ package net.sergeych.bintools
  * like multiplatform version of DataInput
  *
  */
+@Suppress("unused")
 interface DataSource {
 
     /**
@@ -15,6 +16,12 @@ interface DataSource {
 
     fun readByte(): Byte
 
+    /**
+     * true if there is no more data available and next read operation will surely
+     * throw EndOfData. Can return null if it is impossible to determine (for some
+     * async sources)
+     */
+    fun isEnd(): Boolean? = null
 
     fun readUByte() = readByte().toUByte()
 
@@ -33,7 +40,7 @@ interface DataSource {
 
     fun readDouble() = Double.fromBits(readI64())
 
-    fun readFloat() = Float.fromBits(readI32()).toFloat()
+    fun readFloat() = Float.fromBits(readI32())
 
     fun readSmartUInt(): UInt = Smartint.decodeUnsigned(this).toUInt()
     fun readSmartInt(): Int = Smartint.decodeSigned(this).toInt()
@@ -44,12 +51,14 @@ interface DataSource {
 
 
 }
-
 fun ByteArray.toDataSource(): DataSource =
     object : DataSource {
         var position = 0
             private set
 
+        @Suppress("RedundantNullableReturnType")
+        override fun isEnd(): Boolean? = position == size
+
         override fun readByte(): Byte =
             if (position < size) this@toDataSource[position++]
             else throw DataSource.EndOfData()
@@ -59,5 +68,23 @@ fun ByteArray.toDataSource(): DataSource =
         }
     }
 
+@Suppress("unused")
+fun UByteArray.toDataSource(): DataSource =
+    object : DataSource {
+        var position = 0
+            private set
+
+        @Suppress("RedundantNullableReturnType")
+        override fun isEnd(): Boolean? = position == size
+
+        override fun readByte(): Byte =
+            if (position < size) this@toDataSource[position++].toByte()
+            else throw DataSource.EndOfData()
+
+        override fun toString(): String {
+            return "ASrc[$position]: ${encodeToHex()}"
+        }
+    }
+
 inline fun <reified T : Any> DataSource.readNumber(): T = Smartint.decode(this) as T
 

src/commonMain/kotlin/net.sergeych.bintools/KVStorage.kt

@@ -0,0 +1,223 @@
+package net.sergeych.bintools
+
+import kotlinx.serialization.serializer
+import net.sergeych.bipack.BipackDecoder
+import net.sergeych.bipack.BipackEncoder
+import kotlin.reflect.KProperty
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+
+/**
+ * Generic storage of binary content. PArsec uses boss encoding to store everything in it
+ * in a convenient way. See [KVStorage.stored], [KVStorage.invoke] and
+ * [KVStorage.optStored] delegates. The [MemoryKVStorage] is an implementation that stores
+ * values in memory, allowing to connect some other (e.g. persistent storage) later in a
+ * completely transparent way. It can also be used to cache values on the fly.
+ *
+ * Also, it is possible to use [read] and [write] where delegated properties
+ * do not fit well.
+ */
+@Suppress("unused")
+interface KVStorage {
+    operator fun get(key: String): ByteArray?
+    operator fun set(key: String, value: ByteArray?)
+
+    /**
+     * Check whether key is in storage.
+     * Default implementation uses [keys]. You may override it for performance
+     */
+    operator fun contains(key: String): Boolean = key in keys
+
+    val keys: Set<String>
+
+
+    /**
+     * Get number of object in the storage
+     * Default implementation uses [keys]. You may override it for performance
+     */
+    val size: Int get() = keys.size
+
+    /**
+     * Clears all objects in the storage
+     * Default implementation uses [keys]. You may override it for performance
+     */
+    fun clear() {
+        for (k in keys) this[k] = null
+    }
+
+    /**
+     * Default implementation uses [keys]. You may override it for performance
+     */
+    fun isEmpty() = size == 0
+
+    /**
+     * Default implementation uses [keys]. You may override it for performance
+     */
+    fun isNotEmpty() = size != 0
+
+    /**
+     * Add all elements from another storage, overwriting any existing
+     * keys.
+     */
+    fun addAll(other: KVStorage) {
+        for (k in other.keys) {
+            this[k] = other[k]
+        }
+    }
+
+    /**
+     * Delete element by key
+     */
+    fun delete(key: String) {
+        set(key, null)
+    }
+}
+
+/**
+ * Write
+ */
+inline fun <reified T>KVStorage.write(key: String,value: T) {
+    this[key] = BipackEncoder.encode<T>(value)
+}
+
+@Suppress("unused")
+inline fun <reified T>KVStorage.save(key: String,value: T?) {
+    if( value != null ) write(key, value)
+    else delete(key)
+}
+
+inline fun <reified T:Any>KVStorage.read(key: String): T? =
+    this[key]?.let { BipackDecoder.decode<T>(it) }
+
+@Suppress("unused")
+inline fun <reified T:Any>KVStorage.load(key: String): T? = read(key)
+
+inline operator fun <reified T> KVStorage.invoke(defaultValue: T,overrideName: String? = null) =
+    KVStorageDelegate<T>(this, typeOf<T>(), defaultValue, overrideName)
+
+inline fun <reified T> KVStorage.stored(defaultValue: T, overrideName: String? = null) =
+    KVStorageDelegate<T>(this, typeOf<T>(), defaultValue, overrideName)
+
+inline fun <reified T> KVStorage.optStored(overrideName: String? = null) =
+    KVStorageDelegate<T?>(this, typeOf<T?>(), null, overrideName)
+
+class KVStorageDelegate<T>(
+    private val storage: KVStorage,
+    type: KType,
+    private val defaultValue: T,
+    private val overrideName: String? = null,
+) {
+
+    private fun name(property: KProperty<*>): String = overrideName ?: property.name
+
+    private var cachedValue: T = defaultValue
+    private var cacheReady = false
+    private val serializer = serializer(type)
+
+    @Suppress("UNCHECKED_CAST")
+    operator fun getValue(thisRef: Any?, property: KProperty<*>): T {
+        if (cacheReady) return cachedValue
+        val data = storage.get(name(property))
+        if (data == null)
+            cachedValue = defaultValue
+        else
+            cachedValue = BipackDecoder.decode(data.toDataSource(), serializer) as T
+        cacheReady = true
+        return cachedValue
+    }
+
+    operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {
+//        if (!cacheReady || value != cachedValue) {
+            cachedValue = value
+            cacheReady = true
+            storage[name(property)] = BipackEncoder.encode(serializer, value)
+//        }
+    }
+}
+
+/**
+ * Memory storage. Allows connecting to another storage (e.g. persistent one) at come
+ * point later in a transparent way.
+ */
+class MemoryKVStorage(copyFrom: KVStorage? = null) : KVStorage {
+
+    // is used when connected:
+    private var underlying: KVStorage? = null
+
+    // is used while underlying is null:
+    private val data = mutableMapOf<String, ByteArray>()
+
+    /**
+     * Connect some other storage. All existing data will be copied to the [other]
+     * storage. After this call all data access will be routed to [other] storage.
+     */
+    @Suppress("unused")
+    fun connectToStorage(other: KVStorage) {
+        other.addAll(this)
+        underlying = other
+        data.clear()
+    }
+
+
+    /**
+     * Get  data from either memory or a connected storage, see [connectToStorage]
+     */
+    override fun get(key: String): ByteArray? {
+        underlying?.let {
+            return it[key]
+        }
+        return data[key]
+    }
+
+    /**
+     * Put data to memory storage or connected storage if [connectToStorage] was called
+     */
+    override fun set(key: String, value: ByteArray?) {
+        underlying?.let { it[key] = value } ?: run {
+            if (value != null) data[key] = value
+            else data.remove(key)
+        }
+    }
+
+    /**
+     * Checks the item exists in the memory storage or connected one, see[connectToStorage]
+     */
+    override fun contains(key: String): Boolean {
+        underlying?.let { return key in it }
+        return key in data
+    }
+
+    override val keys: Set<String>
+        get() = underlying?.keys ?: data.keys
+
+    override fun clear() {
+        underlying?.clear() ?: data.clear()
+    }
+
+    init {
+        copyFrom?.let { addAll(it) }
+    }
+}
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+expect fun defaultNamedStorage(name: String): KVStorage

src/commonMain/kotlin/net.sergeych.bintools/MRUCache.kt

@@ -0,0 +1,64 @@
+package net.sergeych.bintools
+
+/**
+ * Most Recently Used keys Cache.
+ * Maintains the specified size, removed least used elements on insertion. Element usage is
+ * when it is inserted, updated or accessed (with [get]). Least recently used (LRU) keys
+ * are automatically removed to maintain the [maxSize].
+ *
+ * Note that the cost, [MRUCache] is slower than [MutableMap].
+ */
+class MRUCache<K,V>(val maxSize: Int,
+                    private val cache: LinkedHashMap<K,V> = LinkedHashMap()
+): MutableMap<K,V> by cache {
+
+    private fun checkSize() {
+        while(cache.size > maxSize) {
+            cache.remove(cache.keys.first())
+        }
+    }
+
+    /**
+     * Put the [value] associated with [key] which becomes MRU whether it existed in the cache or was added now.
+     *
+     * If [size] == [maxSize] LRU key will be dropped.
+     *
+     * @return old value for the [key] or null
+     */
+    override fun put(key: K, value: V): V? {
+        // we need it to become MRU, so we remove it to clear its position
+        val oldValue = cache.remove(key)
+        // now we always add, not update, so it will become MRU element:
+        cache.put(key,value).also { checkSize() }
+        return oldValue
+    }
+
+    /**
+     * Put all the key-value pairs, this is exactly same as calling [put] in the same
+     * order. Note that is the [from] map is not linked and its size is greater than
+     * [maxSize], some unpredictable keys will not be added. To be exact, only last
+     * [maxSize] keys will be added by the order providing by [from] map entries
+     * enumerator.
+     *
+     * If from is [LinkedHashMap] or like, onl
+     */
+    override fun putAll(from: Map<out K, V>) {
+        // maybe we should optimize it not to add unnecessary first keys
+        for( e in from) {
+            put(e.key,e.value)
+            checkSize()
+        }
+    }
+
+    /**
+     * Get the value associated with the [key]. It makes the [key] a MRU (last to delete)
+     */
+    override fun get(key: K): V? {
+        return cache[key]?.also {
+            cache.remove(key)
+            cache[key] = it
+        }
+    }
+
+    override fun toString(): String = cache.toString()
+}

src/commonMain/kotlin/net.sergeych.bintools/MotherPacker.kt

@@ -24,6 +24,7 @@ class JsonPacker : MotherPacker {
         return Json.encodeToString(serializer(type), payload).encodeToByteArray()
     }
 
+    @Suppress("UNCHECKED_CAST")
     override fun <T> unpack(type: KType, packed: ByteArray): T {
         return Json.decodeFromString<T>(
             serializer(type) as KSerializer<T>,

src/commonMain/kotlin/net.sergeych.bintools/simple_codecs.kt

@@ -106,12 +106,15 @@ fun UByte.encodeToHex(length: Int = 0) = toLong().encodeToHex(length)
 fun ULong.encodeToHex(length: Int = 0) = toLong().encodeToHex(length)
 
 fun ByteArray.encodeToHex(separator: String = " "): String = joinToString(separator) { it.toUByte().encodeToHex(2) }
+fun UByteArray.encodeToHex(separator: String = " "): String = joinToString(separator) { it.encodeToHex(2) }
 
 @Suppress("unused")
 fun Collection<Byte>.encodeToHex(separator: String = " "): String = joinToString(separator) { it.toUByte().encodeToHex(2) }
 
 fun ByteArray.toDump(wide: Boolean = false): String = toDumpLines(wide).joinToString("\n")
 
+fun UByteArray.toDump(wide: Boolean = false): String = asByteArray().toDumpLines(wide).joinToString("\n")
+
 fun ByteArray.toDumpLines(wide: Boolean = false): List<String> {
 
     val lineSize = if (wide) 32 else 16

src/commonMain/kotlin/net.sergeych.bipack/BipackDecoder.kt

@@ -3,6 +3,7 @@ package net.sergeych.bipack
 import kotlinx.datetime.Instant
 import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.KSerializer
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.encoding.AbstractDecoder
@@ -31,18 +32,20 @@ class BipackDecoder(
     override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
     override fun decodeByte(): Byte = input.readByte()
     override fun decodeShort(): Short =
-        if( fixedNumber ) input.readI16()
+        if (fixedNumber) input.readI16()
         else if (nextIsUnsigned)
             input.readNumber<UInt>().toShort()
         else
             input.readNumber()
+
     override fun decodeInt(): Int =
         if (fixedNumber) input.readI32()
         else if (nextIsUnsigned) input.readNumber<UInt>().toInt() else input.readNumber()
 
     override fun decodeLong(): Long =
-        if( fixedNumber ) input.readI64()
+        if (fixedNumber) input.readI64()
         else if (nextIsUnsigned) input.readNumber<ULong>().toLong() else input.readNumber()
+
     override fun decodeFloat(): Float = input.readFloat()
     override fun decodeDouble(): Double = input.readDouble()
     override fun decodeChar(): Char = Char(input.readNumber<UInt>().toInt())
@@ -57,7 +60,8 @@ class BipackDecoder(
     override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readNumber<UInt>().toInt()
 
     override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        if (elementIndex >= elementsCount) return CompositeDecoder.DECODE_DONE
+        if (elementIndex >= elementsCount)
+            return CompositeDecoder.DECODE_DONE
         nextIsUnsigned = false
         for (a in descriptor.getElementAnnotations(elementIndex)) {
             when (a) {
@@ -70,11 +74,13 @@ class BipackDecoder(
     }
 
     override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
-        return if( deserializer == Instant.serializer() )
+        return if (deserializer == Instant.serializer())
             Instant.fromEpochMilliseconds(decodeLong()) as T
         else
             super.decodeSerializableValue(deserializer)
     }
+
+
     override fun decodeSequentially(): Boolean = isCollection
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder {
@@ -121,7 +127,11 @@ class BipackDecoder(
         super.endStructure(descriptor)
     }
 
-    override fun decodeNotNullMark(): Boolean = decodeBoolean()
+    override fun decodeNotNullMark(): Boolean = try {
+        decodeBoolean()
+    } catch (_: DataSource.EndOfData) {
+        false
+    }
 
     @ExperimentalSerializationApi
     override fun decodeNull(): Nothing? = null
@@ -134,7 +144,16 @@ class BipackDecoder(
         inline fun <reified T> decode(source: DataSource): T = decode(source, serializer())
         inline fun <reified T> decode(source: ByteArray): T =
             decode(source.toDataSource(), serializer())
+        fun <T> decode(serializer: KSerializer<T>, source: ByteArray): T =
+            decode(source.toDataSource(), serializer)
+        inline fun <reified T> decode(source: UByteArray): T =
+            decode(source.toDataSource(), serializer())
+        fun <T> decode(serializer: KSerializer<T>, source: UByteArray): T =
+            decode(source.toDataSource(), serializer)
     }
 }
 
 inline fun <reified T> ByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
+@Suppress("unused")
+inline fun <reified T> UByteArray.decodeFromBipack() = BipackDecoder.decode<T>(this)
+

src/commonMain/kotlin/net.sergeych.bipack/BipackEncoder.kt

@@ -126,3 +126,8 @@ class BipackEncoder(val output: DataSink) : AbstractEncoder() {
 
     }
 }
+
+@Suppress("unused")
+inline fun <reified T> toBipackUByteArray(value: T): UByteArray = BipackEncoder.encode(value).toUByteArray()
+@Suppress("unused")
+inline fun <reified T> toBipackByteArray(value: T): ByteArray = BipackEncoder.encode(value)

src/commonMain/kotlin/net.sergeych.bipack/MotherBipack.kt

@@ -6,11 +6,13 @@ import net.sergeych.bintools.MotherPacker
 import net.sergeych.bintools.toDataSource
 import kotlin.reflect.KType
 
+@Suppress("unused")
 class MotherBipack : MotherPacker {
     override fun <T> pack(type: KType, payload: T): ByteArray {
         return BipackEncoder.encode(serializer(type), payload)
     }
 
+    @Suppress("UNCHECKED_CAST")
     override fun <T> unpack(type: KType, packed: ByteArray): T {
         return BipackDecoder.decode<T>(packed.toDataSource(),
             serializer(type) as KSerializer<T>)

src/commonMain/kotlin/net.sergeych.bipack/annotations.kt

@@ -4,9 +4,16 @@ import kotlinx.serialization.SerialInfo
 
 /**
  * If this annotation is presented in some @Serializable class definition, its instances
- * will be serialized with leading number of fields. This allows to extend class later
+ * will be serialized with the leading number of fields. This allows extending class later
  * providing new parameters __to the end of the class__ and _with default values__.
  *
+ * __IMPORTANT NOTE__. Since version 0.0.7 it's been also possible to use default values
+ * for non-serialized fields after the end-of-data. If the source reports it correctly, e.g.
+ * [net.sergeych.bintools.DataSource.isEnd] returns true, the unset fields are initialized
+ * with default value. This approach ___is not working when the loading instance is not the last
+ * in the deciding array!___, still it is useful to decode isolated objects. We recommend to
+ * use [Extendable] where needed and possible.
+ *
  * Whe deserializing such instances from previous version binaries, the new parameters
  * will get default values.
  *

src/commonMain/kotlin/net/sergeych/synctools/AtomicCounter.kt

@@ -0,0 +1,15 @@
+package net.sergeych.synctools
+
+/**
+ * Thread-safe multiplatform counter
+ */
+@Suppress("unused")
+class AtomicCounter(initialValue: Long = 0) : AtomicValue<Long>(initialValue) {
+
+    fun incrementAndGet(): Long = op { ++actualValue }
+    fun getAndIncrement(): Long = op { actualValue++ }
+
+    fun decrementAndGet(): Long = op { --actualValue }
+
+    fun getAndDecrement(): Long = op { actualValue-- }
+}

src/commonMain/kotlin/net/sergeych/synctools/AtomicValue.kt

@@ -0,0 +1,37 @@
+package net.sergeych.synctools
+
+/**
+ * Multiplatform (JS and battery included) atomically mutable value.
+ * Actual value can be either changed in a block of [mutate] when
+ * new value _depends on the current value_ or use a same [value]
+ * property that is thread-safe where there are threads and just safe
+ * otherwise ;)
+ */
+open class AtomicValue<T>(initialValue: T) {
+    var actualValue = initialValue
+        protected set
+
+    protected val op = ProtectedOp()
+
+    /**
+     * Change the value: get the current and set to the returned, all in the
+     * atomic operation. All other mutating requests including assigning to [value]
+     * will be blocked and queued.
+     * @return result of the mutation. Note that immediate call to property [value]
+     *      could already return modified bu some other thread value!
+     */
+    fun mutate(mutator: (T) -> T): T = op {
+        actualValue = mutator(actualValue)
+        actualValue
+    }
+
+    /**
+     * Atomic get or set the value. Atomic get means if there is a [mutate] in progress
+     * it will wait until the mutation finishes and then return the correct result.
+     */
+    var value: T
+        get() = op { actualValue }
+        set(value) {
+            mutate { value }
+        }
+}

src/commonMain/kotlin/net/sergeych/synctools/ProtectedOp.kt

@@ -0,0 +1,60 @@
+package net.sergeych.synctools
+
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+
+/**
+ * Multiplatform interface to perform a regular (not suspend) operation
+ * protected by a platform mutex (where necessary). Get real implementation
+ * with [ProtectedOp] and use it with [ProtectedOpImplementation.withLock] and
+ * [ProtectedOpImplementation.invoke]
+ */
+interface ProtectedOpImplementation {
+    /**
+     * Get a lock. Be sure to release it.
+     * The recommended way is using [ProtectedOpImplementation.withLock] and
+     * [ProtectedOpImplementation.invoke]
+     */
+    fun lock()
+
+    /**
+     * Release a lock.
+     * The recommended way is using [ProtectedOpImplementation.withLock] and
+     * [ProtectedOpImplementation.invoke]
+     */
+    fun unlock()
+}
+
+@ExperimentalContracts
+inline fun <T> ProtectedOpImplementation.withLock(f: () -> T): T {
+    contract {
+        callsInPlace(f, InvocationKind.EXACTLY_ONCE)
+    }
+    lock()
+    return try {
+        f()
+    } finally {
+        unlock()
+    }
+}
+
+/**
+ * Run a block mutualy-exclusively, see [ProtectedOp]
+ */
+operator fun <T> ProtectedOpImplementation.invoke(f: () -> T): T = withLock(f)
+
+/**
+ * Get the platform-depended implementation of a mutex. It does nothing in the
+ * browser and use appropriate mechanics on JVM and native targets. See
+ * [ProtectedOpImplementation.invoke], [ProtectedOpImplementation.withLock]
+ * ```kotlin
+ * val op = ProtectedOp()
+ * //...
+ * op {
+ *      // mutually exclusive execution
+ *      println("sequential execution here")
+ * }
+ * ~~~
+ */
+expect fun ProtectedOp(): ProtectedOpImplementation

src/commonMain/kotlin/net/sergeych/synctools/WaitHandle.kt

@@ -0,0 +1,23 @@
+package net.sergeych.synctools
+
+/**
+ * Platform-independent interface to thread wait/notify. Does nothing in JS/browser,
+ * and uses appropriate mechanics on other platforms.
+ */
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+expect class WaitHandle() {
+
+    /**
+     * Wait for [wakeUp] as long as [milliseconds] milliseconds, or forever.
+     * Notice it returns immediately on the single-theaded platforms like JS.
+     * @param milliseconds to wait, use 0 to wait indefinitely
+     * @return true if [wakeUp] was called before the timeout, false on timeout.
+     */
+    fun await(milliseconds: Long=0): Boolean
+
+    /**
+     * Awake all [await]'ing threads. Does nothing in the single-threaded JS
+     * environment
+     */
+    fun wakeUp()
+}

src/commonTest/kotlin/bintools/CrcTest.kt

@@ -0,0 +1,16 @@
+package bintools
+
+import net.sergeych.bintools.CRC
+import net.sergeych.bintools.encodeToHex
+import kotlin.test.Test
+
+class CrcTest {
+    @Test
+    fun testVectors() {
+        val x = byteArrayOf(1,2,3,4,5)
+        val crc = CRC.crc8(x)
+        println("->> ${x.toList()}")
+        println("->> ${crc.encodeToHex()}")
+        println("->> ${crc}")
+    }
+}

src/commonTest/kotlin/bintools/TestTools.kt

@@ -1,7 +1,11 @@
 package bintools
 
+import net.sergeych.bintools.MRUCache
 import net.sergeych.bintools.toDump
 import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
 
 class TestTools {
     @Test
@@ -16,4 +20,42 @@ class TestTools {
             println(res.toDump())
         }
     }
+
+    @Test
+    fun testCache() {
+        val cache = MRUCache<Int,String>(3)
+        cache.putAll( mapOf(1 to "one", 2 to "two", 3 to "three", 4 to "four" ) )
+        assertNull(cache[0])
+
+        // this actually should reset MRU for 2:
+        assertEquals("two", cache[2])
+        assertNull(cache[1])
+        assertEquals(3, cache.size)
+        assertTrue { 3 in cache }
+        assertTrue { 4 in cache }
+
+    // now MRU is 2 (checked in assertEquals above) so LRU to drop is 3!
+        cache[5] = "five"
+        assertEquals(3, cache.size)
+        assertTrue { 2 in cache }
+        assertTrue { 4 in cache }
+        assertTrue { 5 in cache }
+
+        cache.getOrPut(3) { "new three"}
+        assertEquals(3, cache.size)
+        assertTrue { 2 in cache }
+        assertTrue { 3 in cache }
+        assertTrue { 5 in cache }
+
+        cache[2] = "New Two"
+        cache[6] = "six"
+
+        // 2 is now second used
+        // amd 6 is MRU, oldest is therefore 5
+
+        assertEquals(3, cache.size)
+        assertTrue { 2 in cache }
+        assertTrue { 6 in cache }
+        assertTrue { 3 in cache }
+    }
 }

src/commonTest/kotlin/bipack/BipackEncoderTest.kt

@@ -30,6 +30,7 @@ data class Foobar2(val bar: Int, val foo: Int, val other: Int = -1)
 @Framed
 data class FoobarF1(val bar: Int, val foo: Int = 117)
 
+@Suppress("unused")
 @Serializable
 @Framed
 @SerialName("bipack.FoobarF1")
@@ -46,6 +47,14 @@ data class FoobarF3(val bar: Int, val foo: Int, val other: Int = -1)
 @CrcProtected()
 data class FoobarFP1(val bar: Int, val foo: Int, val other: Int = -1)
 
+@Serializable
+sealed class SC1 {
+
+    @Serializable
+    class Nested : SC1()
+}
+
+@Suppress("unused")
 class BipackEncoderTest {
 
     @Serializable
@@ -349,7 +358,7 @@ class BipackEncoderTest {
     @Test
     fun testInstant() {
         val x = Clock.System.now()
-//        println( BipackEncoder.encode(x).toDump() )
+        println(BipackEncoder.encode(x).toDump())
         val y = BipackDecoder.decode<Instant>(BipackEncoder.encode(x))
         assertEquals(x.toEpochMilliseconds(), y.toEpochMilliseconds())
     }
@@ -362,7 +371,7 @@ class BipackEncoderTest {
         @Fixed
         val i: UInt,
         @Fixed
-        val li: ULong
+        val li: ULong,
     )
 
     @Serializable
@@ -373,27 +382,80 @@ class BipackEncoderTest {
         @Unsigned
         val i: UInt,
         @Unsigned
-        val li: ULong
+        val li: ULong,
     )
 
     @Test
     fun vectors() {
-        val x = UInts(7u, 64000.toUShort(), 66000u, 931127140399u);
+        val x = UInts(7u, 64000.toUShort(), 66000u, 931127140399u)
-        val p = BipackEncoder.encode(x);
+        val p = BipackEncoder.encode(x)
         println(p.toDump())
         println(p.encodeToBase64Compact())
         val y = BipackDecoder.decode<UInts>(p)
         assertEquals(x, y)
 
-        val xv = VarUInts(7u, 64000.toUShort(), 66000u, 931127140399u);
+        val xv = VarUInts(7u, 64000.toUShort(), 66000u, 931127140399u)
-        val pv = BipackEncoder.encode(xv);
+        val pv = BipackEncoder.encode(xv)
         println(pv.toDump())
         println(pv.encodeToBase64Compact())
         val yv = BipackDecoder.decode<VarUInts>(pv)
         assertEquals(xv, yv)
         println(xv)
         println(yv)
+    }
+
+    @Test
+    fun testStrangeUnpack() {
+        @Serializable
+        data class SFoo(val code: Int, val s1: String? = null, val s2: String? = null)
+
+        val z = BipackEncoder.encode(117)
+        println(z.toDump())
+        val sf = BipackDecoder.decode<SFoo>(z)
+        println(sf)
+    }
+
+    @Serializable
+    enum class TU1 {
+        N1, N2, N3, N4
+    }
 
 
+    @Test
+    fun testUnsignedEnums() {
+        val p1 = BipackEncoder.encode(TU1.N4)
+//        val p2 = BipackEncoder.encode(3u)
+        println(p1.toDump())
+//        println(p2.toDump())
+        val t2 = BipackDecoder.decode<TU1>(p1)
+        assertEquals(TU1.N4, t2)
+        assertEquals(0x0cu, p1[0].toUByte())
+    }
+
+    @Test
+    fun testClosedSerialization() {
+        val x: SC1 = SC1.Nested()
+        val b = BipackEncoder.encode(x)
+        println(b.toDump())
+        val y = BipackDecoder.decode<SC1>(b)
+        println(y)
+    }
+
+
+    @Serializable
+    data class T1(@Fixed val i: Byte)
+
+    @Test
+    fun testFixedByte() {
+        fun t1(i: Int) {
+            val packed = BipackEncoder.encode(T1(i.toByte()))
+            println(packed.toDump())
+            assertEquals(1, packed.size)
+            assertEquals(i, BipackDecoder.decode<T1>(packed).i.toInt())
+        }
+        t1(127)
+        t1(-127)
+        t1(1)
+        t1(-1)
     }
 }

src/commonTest/kotlin/synctools/AtomicCounterTest.kt

@@ -0,0 +1,20 @@
+package net.sergeych.synctools
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class AtomicCounterTest {
+    @Test
+    fun incrementAndDecrement() {
+        val ac = AtomicCounter(7)
+        assertEquals(7, ac.getAndIncrement())
+        assertEquals(8, ac.value)
+        assertEquals(9, ac.incrementAndGet())
+        assertEquals(9, ac.value)
+
+        assertEquals(9, ac.getAndDecrement())
+        assertEquals(8, ac.value)
+        assertEquals(7, ac.decrementAndGet())
+        assertEquals(7, ac.value)
+    }
+}

src/iosArm64Main/kotlin/net.sergeych.bintools/KVStorage.iosArm64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/iosArm64Main/kotlin/net/sergeych/synctools/ProtectedOp.iosArm64.kt

@@ -0,0 +1,27 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Get the platform-depended implementation of a mutex. It does nothing in the
+ * browser and use appropriate mechanics on JVM and native targets. See
+ * [ProtectedOpImplementation.invoke], [ProtectedOpImplementation.withLock]
+ * ```kotlin
+ * val op = ProtectedOp()
+ * //...
+ * op {
+ *      // mutually exclusive execution
+ *      println("sequential execution here")
+ * }
+ * ~~~
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/iosArm64Main/kotlin/net/sergeych/synctools/WaitHandle.iosArm64.kt

@@ -0,0 +1,38 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+/**
+ * Platform-independent interface to thread wait/notify. Does nothing in JS/browser,
+ * and uses appropriate mechanics on other platforms.
+ */
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/iosSimulatorArm64Main/kotlin/net.sergeych.bintools/KVStorage.iosSimulatorArm64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/iosSimulatorArm64Main/kotlin/net/sergeych/synctools/ProtectedOp.iosSimulatorArm64.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/iosSimulatorArm64Main/kotlin/net/sergeych/synctools/WaitHandle.iosSimulatorArm64.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/iosX64Main/kotlin/net.sergeych.bintools/KVStorage.iosX64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/iosX64Main/kotlin/net/sergeych/synctools/ProtectedOp.iosX64.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/iosX64Main/kotlin/net/sergeych/synctools/WaitHandle.iosX64.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/jsMain/kotlin/net.sergeych.bintools/KVStorage.js.kt

@@ -0,0 +1,42 @@
+package net.sergeych.bintools
+
+import kotlinx.browser.localStorage
+import net.sergeych.mp_tools.decodeBase64Compact
+import net.sergeych.mp_tools.encodeToBase64Compact
+import org.w3c.dom.Storage
+import org.w3c.dom.set
+
+actual fun defaultNamedStorage(name: String): KVStorage =  BrowserKVStorage(name, localStorage)
+
+/**
+ * Default KV storage in browser. Use if with `localStorage` or `sessionStorage`. It uses
+ * prefix for storage values not to collide with other data, beware of it.
+ */
+class BrowserKVStorage(keyPrefix: String, private val bst: Storage) : KVStorage {
+
+    private val prefix = "$keyPrefix:"
+    fun k(key: String) = "$prefix$key"
+    override fun get(key: String): ByteArray? {
+        return bst.getItem(k(key))?.decodeBase64Compact()
+    }
+
+    override fun set(key: String, value: ByteArray?) {
+        val corrected = k(key)
+        if (value == null)
+            bst.removeItem(corrected)
+        else
+            bst.set(corrected, value.encodeToBase64Compact())
+    }
+
+    override val keys: Set<String>
+        get() {
+            val kk = mutableListOf<String>()
+            for (i in 0 until bst.length) {
+                val k = bst.key(i) ?: break
+                if( k.startsWith(prefix)) {
+                    kk += k.substring(prefix.length)
+                }
+            }
+            return kk.toSet()
+        }
+}

src/jsMain/kotlin/net/sergeych/synctools/ProtectedOp.js.kt

@@ -0,0 +1,9 @@
+package net.sergeych.synctools
+
+/**
+ * JS is single-threaded, so we don't need any additional protection:
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    override fun lock() {}
+    override fun unlock() {}
+}

src/jsMain/kotlin/net/sergeych/synctools/WaitHandle.js.kt

@@ -0,0 +1,13 @@
+package net.sergeych.synctools
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    actual fun await(milliseconds: Long): Boolean {
+        // in JS we can't wait: no threads
+        return true
+    }
+
+    actual fun wakeUp() {
+        // in JS we can't wait: no threads
+    }
+}

src/jvmMain/kotlin/net.sergeych.bintools/FileDataProvider.kt

@@ -0,0 +1,59 @@
+package net.sergeych.bintools
+
+import java.io.InputStream
+import java.io.OutputStream
+import java.nio.file.Path
+import kotlin.io.path.*
+
+class FileDataProvider(val folder: Path) : DataProvider {
+
+    class Source(private val input: InputStream) : DataSource {
+        override fun readByte(): Byte {
+            val b = input.read()
+            if (b < 0) throw DataSource.EndOfData()
+            return b.toByte()
+        }
+
+        override fun readBytes(size: Int): ByteArray {
+            return input.readNBytes2(size).also {
+                if (it.size < size) throw DataSource.EndOfData()
+            }
+        }
+    }
+
+    class Sink(private val out: OutputStream) : DataSink {
+        override fun writeByte(data: Byte) {
+            out.write(data.toInt())
+        }
+    }
+
+    override fun <T> read(name: String, f: (DataSource) -> T): T = folder.resolve(name).inputStream().buffered().use {
+        f(Source(it))
+    }
+
+    override fun write(name: String, f: (DataSink) -> Unit) {
+        folder.resolve(name).outputStream().use { f(Sink(it)) }
+    }
+
+    override fun delete(name: String) {
+        println("file: $folder -- $name")
+        folder.resolve(name).deleteExisting()
+    }
+
+    override fun list(): List<String> = folder
+        .listDirectoryEntries()
+        .filter { it.isRegularFile() && it.isReadable() }
+        .map { it.name }
+}
+
+/**
+ * Compatibility with Java8 and android. Read up to N bytes
+ */
+fun InputStream.readNBytes2(size: Int): ByteArray {
+    val result = ByteArray(size)
+    val len = read(result)
+    return if (len < size)
+        result.sliceArray(0..<len)
+    else
+        result
+}

src/jvmMain/kotlin/net.sergeych.bintools/KVStorage.jvm.kt

@@ -0,0 +1,21 @@
+package net.sergeych.bintools
+
+import java.nio.file.Paths
+import kotlin.io.path.createDirectories
+
+actual fun defaultNamedStorage(name: String): KVStorage {
+    val rootFolder = Paths.get(when {
+        // absolute path
+        name.startsWith("/") -> name
+        // path - assume the caller knows what to do
+        name.contains("/") -> name
+        // simple name - we will create it in the user home:
+        else -> {
+            val home = System.getProperty("user.home")
+            "$home/.local_storage/$name"
+        }
+    })
+    rootFolder.createDirectories()
+    val provider = FileDataProvider(rootFolder)
+    return DataKVStorage(provider)
+}

src/jvmMain/kotlin/net/sergeych/synctools/ContinuationHandler.kt

@@ -0,0 +1,34 @@
+@file:Suppress("unused")
+
+package net.sergeych.synctools
+
+import java.nio.channels.CompletionHandler
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+
+/**
+ * Helper class to handle Java continuation with Kotlin coroutines.
+ * Usage sample:
+ * ```kotlin
+ * val socket = withContext(Dispatchers.IO) {
+ *     AsynchronousSocketChannel.open()
+ * }
+ * suspendCoroutine { cont ->
+ *     socket.connect(address.socketAddress, cont, VoidCompletionHandler)
+ * }
+ * ```
+ */
+open class ContinuationHandler<T> : CompletionHandler<T, Continuation<T>> {
+    override fun completed(result: T, attachment: Continuation<T>) {
+        attachment.resume(result)
+    }
+
+    override fun failed(exc: Throwable, attachment: Continuation<T>) {
+        attachment.resumeWithException(exc)
+    }
+}
+
+object VoidCompletionHandler : ContinuationHandler<Void>()
+
+object IntCompletionHandler : ContinuationHandler<Int>()

src/jvmMain/kotlin/net/sergeych/synctools/ProtectedOp.jvm.kt

@@ -0,0 +1,19 @@
+package net.sergeych.synctools
+
+import java.util.concurrent.locks.ReentrantLock
+
+/**
+ * Get the platform-depended implementation of a mutex-protected operation.
+ * JVM version uses a concealed object synchronization pattern (per-object monitor lock)
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+
+}

src/jvmMain/kotlin/net/sergeych/synctools/WaitHandle.jvm.kt

@@ -0,0 +1,22 @@
+package net.sergeych.synctools
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val access = Object()
+
+    actual fun await(milliseconds: Long): Boolean {
+        return synchronized(access) {
+            try {
+                access.wait(milliseconds)
+                true
+            }
+            catch(_: InterruptedException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        synchronized(access) { access.notifyAll() }
+    }
+}

src/jvmTest/kotlin/net/sergeych/bintools/KVStorage_jvmKtTest.kt

@@ -0,0 +1,39 @@
+package net.sergeych.bintools
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+class KVStorage_jvmKtTest {
+    @Test
+    fun testFileStorage() {
+        val s1 = defaultNamedStorage("test_mp_bintools")
+
+        for( n in s1.keys.toList()) s1.delete(n)
+
+        assertTrue(s1.keys.isEmpty())
+        var foo by s1("unknown")
+        assertEquals(foo, "unknown")
+        foo = "bar"
+        assertEquals(foo, "bar")
+        var answer by s1.optStored<Int>()
+        assertNull(answer)
+        answer = 42
+        assertEquals(answer, 42)
+        answer = 43
+
+        println("----------------------------------------------------------------")
+        val s2 = defaultNamedStorage("test_mp_bintools")
+        val foo1 by s2.stored("?", "foo")
+        val answer1: Int? by s2.optStored("answer")
+
+        assertEquals("bar", foo1)
+        assertEquals(43, answer1)
+
+        for( i in 0..< 13 ) {
+            s2.write("test_$i", "payload_$i")
+        }
+
+    }
+}

src/macosArm64Main/kotlin/net.sergeych.bintools/KVStorage.macosArm64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/macosArm64Main/kotlin/net/sergeych/synctools/ProtectedOp.macosArm64.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/macosArm64Main/kotlin/net/sergeych/synctools/WaitHandle.macosArm64.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/macosX64Main/kotlin/net.sergeych.bintools/KVStorage.macosX64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/macosX64Main/kotlin/net/sergeych/synctools/ProtectedOp.macosX64.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/macosX64Main/kotlin/net/sergeych/synctools/WaitHandle.macosX64.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/mingwX64Main/kotlin/net.sergeych.bintools/KVStorage.mingwX64.kt

@@ -0,0 +1,25 @@
+package net.sergeych.bintools
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/mingwX64Main/kotlin/net/sergeych/synctools/ProtectedOp.mingwX64.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/mingwX64Main/kotlin/net/sergeych/synctools/WaitHandle.mingwX64.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/nativeMain/kotlin/net.sergeych.bintools/KVStorage.native.kt

@@ -0,0 +1,5 @@
+package net.sergeych.bintools
+
+actual fun defaultNamedStorage(name: String): KVStorage {
+    TODO("Not yet implemented")
+}

src/nativeMain/kotlin/net/sergeych/synctools/ProtectedOp.native.kt

@@ -0,0 +1,17 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Native implementation uses `ReentrantLock`]
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object : ProtectedOpImplementation {
+    private val access = ReentrantLock()
+    override fun lock() {
+        access.lock()
+    }
+
+    override fun unlock() {
+        access.unlock()
+    }
+}

src/nativeMain/kotlin/net/sergeych/synctools/WaitHandle.native.kt

@@ -0,0 +1,34 @@
+package net.sergeych.synctools
+
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    private val channel = Channel<Unit>()
+    actual fun await(milliseconds: Long): Boolean {
+        return runBlocking {
+            try {
+                if( milliseconds > 0) {
+                    withTimeout(milliseconds) {
+                        channel.receive()
+                        true
+                    }
+                }
+                else {
+                    channel.receive()
+                    true
+                }
+            }
+            catch(_: TimeoutCancellationException) {
+                false
+            }
+        }
+    }
+
+    actual fun wakeUp() {
+        runBlocking { channel.send(Unit) }
+    }
+}

src/wasmJsMain/kotlin/net.sergeych.bintools/KVStorage.wasmJs.kt

@@ -0,0 +1,63 @@
+package net.sergeych.bintools
+
+import kotlinx.browser.localStorage
+import net.sergeych.mp_tools.decodeBase64Compact
+import net.sergeych.mp_tools.encodeToBase64Compact
+import org.w3c.dom.Storage
+import org.w3c.dom.set
+
+
+/**
+ * Create per-platform default named storage.
+ *
+ * - In the browser, it uses the `Window.localStorage` prefixing items
+ *   by a string containing the [name]
+ *
+ * - In the JVM environment it uses folder-based storage on the file system. The name
+ *   is considered to be a folder name (the whole path which will be automatically created)
+ *   using the following rules:
+ *    - when the name starts with slash (`/`) it is treated as an absolute path to a folder
+ *    - when the name contains slash, it is considered to be a relative folder to the
+ *      `User.home` directory, like "`~/`" on unix systems.
+ *    - otherwise, the folder will be created in "`~/.local_storage`" parent directory
+ *      (which also will be created if needed).
+ *
+ *  - For the native platorms it is not yet implemented (but will be soon).
+ *
+ *  See [DataKVStorage] and [DataProvider] to implement a KVStorage on filesystems and like,
+ *  and `FileDataProvider` class on JVM target.
+ */
+actual fun defaultNamedStorage(name: String): KVStorage =  BrowserKVStorage(name, localStorage)
+
+/**
+ * Default KV storage in browser. Use if with `localStorage` or `sessionStorage`. It uses
+ * prefix for storage values not to collide with other data, beware of it.
+ */
+class BrowserKVStorage(keyPrefix: String, private val bst: Storage) : KVStorage {
+
+    private val prefix = "$keyPrefix:"
+    fun k(key: String) = "$prefix$key"
+    override fun get(key: String): ByteArray? {
+        return bst.getItem(k(key))?.decodeBase64Compact()
+    }
+
+    override fun set(key: String, value: ByteArray?) {
+        val corrected = k(key)
+        if (value == null)
+            bst.removeItem(corrected)
+        else
+            bst.set(corrected, value.encodeToBase64Compact())
+    }
+
+    override val keys: Set<String>
+        get() {
+            val kk = mutableListOf<String>()
+            for (i in 0 until bst.length) {
+                val k = bst.key(i) ?: break
+                if( k.startsWith(prefix)) {
+                    kk += k.substring(prefix.length)
+                }
+            }
+            return kk.toSet()
+        }
+}

src/wasmJsMain/kotlin/net/sergeych/synctools/ProtectedOp.wasmJs.kt

@@ -0,0 +1,27 @@
+package net.sergeych.synctools
+
+import kotlinx.atomicfu.locks.ReentrantLock
+
+/**
+ * Get the platform-depended implementation of a mutex. It does nothing in the
+ * browser and use appropriate mechanics on JVM and native targets. See
+ * [ProtectedOpImplementation.invoke], [ProtectedOpImplementation.withLock]
+ * ```kotlin
+ * val op = ProtectedOp()
+ * //...
+ * op {
+ *      // mutually exclusive execution
+ *      println("sequential execution here")
+ * }
+ * ~~~
+ */
+actual fun ProtectedOp(): ProtectedOpImplementation = object: ProtectedOpImplementation {
+    val rlock = ReentrantLock()
+    override fun lock() {
+        rlock.lock()
+    }
+
+    override fun unlock() {
+        rlock.unlock()
+    }
+}

src/wasmJsMain/kotlin/net/sergeych/synctools/WaitHandle.wasmJs.kt

@@ -0,0 +1,19 @@
+package net.sergeych.synctools
+
+/**
+ * Platform-independent interface to thread wait/notify. Does nothing in JS/browser,
+ * and uses appropriate mechanics on other platforms.
+ *
+ * Wasm is effictively songle threaded at the moment
+ */
+@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
+actual class WaitHandle {
+    actual fun await(milliseconds: Long): Boolean {
+        // in JS we can't wait: no threads
+        return true
+    }
+
+    actual fun wakeUp() {
+        // in JS we can't wait: no threads
+    }
+}

src/wasmJsTest/kotlin/StorageTest.kt

@@ -0,0 +1,38 @@
+import net.sergeych.bintools.*
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+class StorageTest {
+
+    @Test
+    fun storageTest() {
+        val s1 = defaultNamedStorage("test_mp_bintools")
+
+        for (n in s1.keys.toList()) s1.delete(n)
+
+        assertTrue(s1.keys.isEmpty())
+        var foo by s1("unknown")
+        assertEquals(foo, "unknown")
+        foo = "bar"
+        assertEquals(foo, "bar")
+        var answer by s1.optStored<Int>()
+        assertNull(answer)
+        answer = 42
+        assertEquals(answer, 42)
+        answer = 43
+
+        val s2 = defaultNamedStorage("test_mp_bintools")
+        val foo1 by s2.stored("?", "foo")
+        val answer1: Int? by s2.optStored("answer")
+
+        assertEquals("bar", foo1)
+        assertEquals(43, answer1)
+
+        for (i in 0..<13) {
+            s2.write("test_$i", "payload_$i")
+        }
+    }
+
+}