initial: varing and smarting packing tools

2023-02-03 13:10:43 +01:00
commit 0f9702b1dc
@@ -0,0 +1,27 @@
+package net.sergeych.bintools
+
+abstract class DataSink {
+
+    abstract fun writeByte(data: Byte)
+
+    fun writeByte(data: Int) = writeByte(data.toByte())
+
+    open fun writeUByte(data: UByte) {
+        writeByte(data.toByte())
+    }
+
+    @Suppress("unused")
+    open fun writeBytes(data: ByteArray) {
+        for(d in data) writeByte(d)
+    }
+}
+
+class ArrayDataSink : DataSink() {
+    private val result = mutableListOf<Byte>()
+
+    override fun writeByte(data: Byte) {
+        result.add(data)
+    }
+
+    fun toByteArray() = result.toByteArray()
+}
@@ -0,0 +1,30 @@
+package net.sergeych.bintools
+
+/**
+ * data input stream-like abstraction. We need it because
+ * kotlinx serialization is synchronous and there us nothing
+ * like multiplatform version of DataInput
+ *
+ */
+abstract class DataSource {
+
+    abstract fun readByte(): Byte
+
+    abstract val position: Int
+
+    open fun readUByte() = readByte().toUByte()
+
+    @Suppress("unused")
+    open fun readBytes(size: Int): ByteArray =
+        ByteArray(size).also { a ->
+            for( i in 0..size)
+                a[i] = readByte()
+        }
+}
+
+fun ByteArray.toDataSource(): DataSource =
+    object : DataSource() {
+        override var position = 0
+
+        override fun readByte(): Byte = this@toDataSource[position++]
+    }
@@ -0,0 +1,4 @@
+package net.sergeych.bintools
+
+class FormatException(message: String="invalid format",cause: Throwable?=null):
+        Exception(message, cause)
@@ -0,0 +1,135 @@
+package net.sergeych.bintools
+
+
+fun longToBytes(value: Long): ByteArray {
+    var l = value
+    val result = ByteArray(8)
+    for (i in 7 downTo 0) {
+        result[i] = (l and 0xFF).toByte()
+        l = l shr 8
+    }
+    return result
+}
+
+/**
+ * Convert 8 bytes to LE long
+ */
+fun bytesToLong(b: ByteArray): Long {
+    var result: Long = 0
+    for (i in 0 until 8) {
+        result = result shl 8
+        result = result or (b[i].toLong() and 0xFF)
+    }
+    return result
+}
+
+
+private val hexDigits = "0123456789ABCDEF"
+
+fun Long.encodeToHex(length: Int = 0): String {
+    var result = ""
+    var value = this
+    if (value < 0) throw IllegalArgumentException("cant convert to hex negative (ambiguous)")
+    do {
+        result = hexDigits[(value and 0x0f).toInt()] + result
+        value = value shr 4
+    } while (value > 0)
+    while (result.length < length) result = "0" + result
+    return result
+}
+
+fun Int.encodeToHex(length: Int = 0) = (toLong() and 0xFFFFffff).encodeToHex(length)
+@Suppress("unused")
+fun UInt.encodeToHex(length: Int = 0) = toLong().encodeToHex(length)
+fun Byte.encodeToHex(length: Int = 0) = (toLong() and 0xFF).encodeToHex(length)
+fun UByte.encodeToHex(length: Int = 0) = toLong().encodeToHex(length)
+@Suppress("unused")
+fun ULong.encodeToHex(length: Int = 0) = toLong().encodeToHex(length)
+
+fun ByteArray.encodeToHex(separator: String = " "): String = joinToString(separator) { it.toUByte().encodeToHex(2) }
+fun Collection<Byte>.encodeToHex(separator: String = " "): String = joinToString(separator) { it.toUByte().encodeToHex(2) }
+
+fun ByteArray.toDump(wide: Boolean = false): String = toDumpLines(wide).joinToString("\n")
+
+fun ByteArray.toDumpLines(wide: Boolean = false): List<String> {
+
+    val lineSize = if (wide) 32 else 16
+
+    fun dumpChars(_from: Int): String {
+        var from = _from
+        val b = StringBuilder(22)
+
+        b.append('|')
+        val max: Int = kotlin.math.min(size, from + lineSize)
+        while (from < max) {
+            val ch = this[from++].toInt()
+            if (ch >= ' '.code && ch < 127) b.append(ch.toChar()) else b.append('.')
+        }
+        val remainder = from % lineSize
+        if (remainder > 0) {
+            var cnt = lineSize - remainder
+            while (cnt-- > 0) b.append(' ')
+        }
+        return b.append("|").toString()
+    }
+
+
+    val lines = mutableListOf<String>()
+    if (size == 0) return lines
+    var line: StringBuilder? = null
+
+    if (size != 0) {
+        for (i in indices) {
+            if (i % lineSize == 0) {
+                if (line != null) {
+                    line.append(dumpChars(i - lineSize))
+                    lines.add(line.toString())
+                }
+                line = StringBuilder(i.encodeToHex(4))
+                line.append(' ')
+            }
+            line!!.append((this[i].toUByte()).encodeToHex(2))
+            line.append(' ')
+        }
+        if (line != null) {
+            val l = size
+            var fill = lineSize - l % lineSize
+            if (fill < lineSize) while (fill-- > 0) line.append("   ")
+            val index = l - l % lineSize
+            line.append(dumpChars(if (index < l) index else l - lineSize))
+            lines.add(line.toString())
+        }
+    }
+    return lines
+}
+
+fun String.decodeHex(): ByteArray {
+    val source = this.trim().uppercase()
+    val result = arrayListOf<Byte>()
+    var pos = 0
+    while (pos < source.length) {
+        val i = hexDigits.indexOf(source[pos++])
+        if (i < 0) throw FormatException("invalid hex digit in ${source} at ${pos - 1}")
+        if (pos >= source.length) throw FormatException(
+            "hex string must consist of bytes " +
+                    "(unexepced end of data): $source"
+        )
+        val j = hexDigits.indexOf(source[pos++])
+        if (j < 0) throw FormatException("invalid hex digit in ${source} at ${pos - 1}")
+        result.add(((i shl 4) or j).toByte())
+        while (pos < source.length && (source[pos].isWhitespace())) pos++
+    }
+    return result.toByteArray()
+}
+
+fun ByteArray.flipSelf() {
+    var i = 0
+    var j = size - 1
+    while (i < j) {
+        val x = this[i]
+        this[i++] = this[j]
+        this[j--] = x
+    }
+}
+
+fun ByteArray.flip(): ByteArray = copyOf().also { it.flipSelf() }
@@ -0,0 +1,150 @@
+@file:OptIn(ExperimentalUnsignedTypes::class)
+
+package com.icodici.ubdata
+
+import net.sergeych.bintools.ArrayDataSink
+import net.sergeych.bintools.DataSink
+import net.sergeych.bintools.DataSource
+import net.sergeych.bintools.toDataSource
+
+/**
+ * Smart variable-length long encoding tools, async.
+ *
+ * | Bytes sz | varint bits | smartint bits |
+ * |:-----:|:------:|:---------:|
+ * |   1   |    7   |     6     |
+ * |   2   |    14  |    14     |
+ * |   3   |    21  |    22     |
+ * |   4   |    28  |    29     |
+ * |   5   |    35  |    36     |
+ * |   6+  |    7*N |   7*N+1   |
+ * |   9   |    63  |   64      |
+ * |   10  |    64  |   ---     |
+ *
+ * In other words, except for very small numbers smartint
+ * gives 1 bit gain. So, full sized 64 bits with smartint takes
+ * 9 bytes, while varint needs 10. This could be important.
+ *
+ * Encoding is the following:
+ *
+ *    Byte 0:  bits 0..1  : type
+ *             bits 2..7 : v0
+ *
+ * Then depending on the type:
+ *
+ *    type = 0:
+ *          v0 is the resul 0..64 (or -32..32)
+ *
+ *    type = 1:
+ *          v0, v1 is the result, 14 bits
+ *
+ *    type = 2:
+ *          v0, v1, v2 are the result, 22bits
+ *
+ *    type = 3:
+ *          v0, v1, v2, varint encoded
+ *
+ *  Varint encodes bytes with last bit reserved as end
+ *  flag and first 7 bits are data bits. Last bit 0 means end of the
+ *  sequence.
+ *
+ */
+object Smartint {
+
+    private val v0limit: ULong = (1L shl 6).toULong()
+    private val v1limit = (1L shl 14).toULong()
+    private val v2limit = (1L shl 22).toULong()
+
+    fun encode(value: ULong, sink: DataSink) {
+        when {
+            value < v0limit -> encodeSeq(sink, 0, value)
+
+            value < v1limit -> encodeSeq(
+                sink,
+                1,
+                value and 0x3Fu,
+                value shr 6
+            )
+
+            value < v2limit -> encodeSeq(
+                sink,
+                2,
+                value and 0x3Fu,
+                (value shr 6) and 0xFFu,
+                (value shr 14) and 0xFFu,
+            )
+
+            else -> {
+                encodeSeq(
+                    sink,
+                    3,
+                    value and 0x3Fu,
+                    (value shr 6) and 0xFFu,
+                    (value shr 14) and 0xFFu,
+                )
+                Varint.encode(value shr 22, sink)
+            }
+        }
+    }
+
+    private fun encodeSeq(sink: DataSink, type: Int, vararg bytes: ULong) {
+        if (bytes.size == 0)
+            sink.writeByte(0)
+        else {
+            if (bytes[0] > v0limit) throw IllegalArgumentException("first number is too big")
+            sink.writeUByte(((type and 0x03) or (bytes[0] shl 2).toInt()).toUByte())
+            for (x in bytes.drop(1))
+                sink.writeUByte(x.toUByte())
+        }
+    }
+
+    fun decode(source: DataSource): ULong {
+        fun get(): ULong = source.readUByte().toULong()
+        val first = get().toUInt()
+        var type = (first and 3u).toInt()
+
+        var result: ULong = first.toULong() shr 2
+        if (type-- == 0) return result // type 0
+
+        result = result or (get() shl 6)
+        if (type-- == 0) return result  // type 1
+
+        result = result or (get() shl 14)
+        if (type == 0) return result  // type 2
+
+        return result or (Varint.decode(source) shl 22)
+    }
+
+    fun encodeSigned(value: Long, sink: DataSink) {
+        val sigBit: ULong
+        val x: ULong
+        if (value < 0) {
+            x = (-value).toULong()
+            sigBit = 1u
+        } else {
+            x = value.toULong()
+            sigBit = 0u
+        }
+        Varint.encode((x shl 1) or sigBit, sink)
+    }
+
+    fun decodeSigned(source: DataSource): Long {
+        val x = Varint.decode(source)
+        val result = (x shr 1).toLong()
+        return if ((x and 1u).toInt() != 0) -result else result
+    }
+
+    fun encode(value: ULong): ByteArray {
+        return ArrayDataSink().also { encode(value, it) }.toByteArray()
+    }
+
+    fun decode(packed: ByteArray) = decode(packed.toDataSource())
+
+    fun encodeSigned(value: Long): ByteArray {
+        return ArrayDataSink().also { encodeSigned(value, it) }.toByteArray()
+    }
+
+    fun decodeSigned(data: ByteArray): Long {
+        return decodeSigned(data.toDataSource())
+    }
+}
@@ -0,0 +1,82 @@
+@file:OptIn(ExperimentalUnsignedTypes::class)
+
+package com.icodici.ubdata
+
+import net.sergeych.bintools.ArrayDataSink
+import net.sergeych.bintools.DataSink
+import net.sergeych.bintools.DataSource
+import net.sergeych.bintools.toDataSource
+
+/**
+ * Variable-length long integer encoding. the MSB (0x80) bit of each byte flags
+ * that it is not the last one, and all ncecssary bits are encoded with 7-bit
+ * portions, LSB to MSB (big endian of sorts).
+ *
+ * There is slower but more compact encoding variant, [Smartint] that is better when
+ * encoding numbers that needs more than 22 bits. With smaller numbers its either
+ * same or even worse, see [Smartint] docs.
+ */
+object Varint {
+    fun encodeSigned(value: Long, sink: DataSink): Unit {
+        var sigBit: ULong
+        var x: ULong
+        if (value < 0) {
+            x = (-value).toULong()
+            sigBit = 1u
+        } else {
+            x = value.toULong()
+            sigBit = 0u
+        }
+        encode((x shl 1) or sigBit, sink)
+    }
+
+    fun encode(value: ULong, dout: DataSink) {
+        var rest = value
+        do {
+            val x = (rest and 127u).toInt()
+            rest = rest shr 7
+            if (rest > 0u)
+                dout.writeByte(x or 0x80)
+            else
+                dout.writeByte(x)
+
+        } while (rest > 0u)
+    }
+
+    fun decode(source: DataSource): ULong {
+        var result: ULong = 0u
+        var count = 0
+        while (true) {
+            val x = source.readUByte().toInt()
+            result = result or ((x and 0x7F).toULong() shl count)
+            if ((x and 0x80) == 0)
+                break
+            count += 7
+        }
+        return result
+    }
+
+
+    fun decodeSigned(source: DataSource): Long {
+        val x = decode(source)
+        val result = (x shr 1).toLong()
+        return if ((x and 1u).toInt() != 0) -result else result
+    }
+
+    fun encode(value: ULong): ByteArray {
+        return ArrayDataSink().also { encode(value, it) }.toByteArray()
+    }
+
+    fun decode(packed: ByteArray) = decode(packed.toDataSource())
+
+    fun encodeSigned(value: Long): ByteArray {
+        return ArrayDataSink().also { encodeSigned(value, it) }.toByteArray()
+    }
+
+    fun decodeSigned(data: ByteArray): Long {
+        return decodeSigned(data.toDataSource())
+    }
+
+
+
+}