refs #35 bit granularity for bitstreams; LZW done

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitInput.kt

@@ -2,10 +2,12 @@ package net.sergeych.lynon
 
 abstract class BitInput {
 
+    data class DataByte(val data: Int,val bits: Int)
+
     /**
      * Return next byte, int in 0..255 range, or -1 if end of stream reached
      */
-    abstract fun getByte(): Int
+    abstract fun getByte(): DataByte
 
     private var accumulator = 0
 
@@ -17,12 +19,13 @@ abstract class BitInput {
     fun getBitOrNull(): Int? {
         if (isEndOfStream) return null
         if (mask == 0) {
-            accumulator = getByte()
+            val ab = getByte()
+            accumulator = ab.data
             if (accumulator == -1) {
                 isEndOfStream = true
                 return null
             }
-            mask = 0x80
+            mask = 1 shl (ab.bits - 1)
         }
         val result = if (0 == accumulator and mask) 0 else 1
         mask = mask shr 1
@@ -31,14 +34,14 @@ abstract class BitInput {
 
     fun getBitsOrNull(count: Int): ULong? {
         var result = 0UL
-        var mask = 1UL
+        var resultMask = 1UL
         for( i in 0 ..< count) {
             when(getBitOrNull()) {
                 null -> return null
-                1 -> result = result or mask
+                1 -> result = result or resultMask
                 0 -> {}
             }
-            mask = mask shl 1
+            resultMask = resultMask shl 1
         }
         return result
     }
@@ -68,6 +71,7 @@ abstract class BitInput {
         return if( isNegative == 1) -value else value
     }
 
+    @Suppress("unused")
     fun getBool(): Boolean {
         return getBit() == 1
     }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/BitOutput.kt

@@ -5,29 +5,45 @@ abstract class BitOutput {
     abstract fun outputByte(byte: UByte)
 
     private var accumulator = 0
+
+    /**
+     * Number of bits in accumulator. After output is closed by [close] this value is
+     * not changed and represents the number of bits in the last byte; this should
+     * be used to properly calculate end of the bit stream
+     */
     private var accumulatorBits = 0
+        private set
+
+    /**
+     * When [close] is called, represents the number of used bits in the last byte;
+     * bits after this number are the garbage and should be ignored
+     */
+    val lastByteBits: Int get() {
+        if( !isClosed ) throw IllegalStateException("BitOutput is not closed")
+        return accumulatorBits
+    }
 
     fun putBits(bits: ULong, count: Int) {
-        require( count <= 64 )
+        require(count <= 64)
         var x = bits
-        for( i in 0 ..< count ) {
-            putBit( (x and 1u).toInt() )
+        for (i in 0..<count) {
+            putBit((x and 1u).toInt())
             x = x shr 1
         }
     }
 
     fun putBits(bits: Int, count: Int) {
-        require( count <= 32 )
+        require(count <= 32)
         var x = bits
-        for( i in 0 ..< count ) {
-            putBit( (x and 1) )
+        for (i in 0..<count) {
+            putBit((x and 1))
             x = x shr 1
         }
     }
 
     fun putBit(bit: Int) {
         accumulator = (accumulator shl 1) or bit
-        if( ++accumulatorBits >= 8 ) {
+        if (++accumulatorBits >= 8) {
             outputByte(accumulator.toUByte())
             accumulator = accumulator shr 0
             accumulatorBits = 0
@@ -38,19 +54,18 @@ abstract class BitOutput {
         val tetrades = sizeInTetrades(value)
         putBits(tetrades - 1, 4)
         var rest = value
-        for( i in 0..<tetrades ) {
-            putBits( rest and 0xFu, 4 )
+        for (i in 0..<tetrades) {
+            putBits(rest and 0xFu, 4)
             rest = rest shr 4
         }
     }
 
     @Suppress("unused")
     fun packSigned(value: Long) {
-        if( value < 0 ) {
+        if (value < 0) {
             putBit(1)
             packUnsigned((-value).toULong())
-        }
-        else {
+        } else {
             putBit(0)
             packUnsigned(value.toULong())
         }
@@ -59,17 +74,18 @@ abstract class BitOutput {
     var isClosed = false
         private set
 
-    fun close() {
-        if( !isClosed ) {
+    fun close(): BitOutput {
+        if (!isClosed) {
             if (accumulatorBits > 0) {
-                while (accumulatorBits != 0) putBit(0)
-            }
+                outputByte(accumulator.toUByte())
+            } else accumulatorBits = 8
             isClosed = true
         }
+        return this
     }
 
     fun putBytes(data: ByteArray) {
-        for( b in data ) {
+        for (b in data) {
             putBits(b.toULong(), 8)
         }
     }

lynglib/src/commonMain/kotlin/net/sergeych/lynon/MemoryBitInput.kt

@@ -1,15 +1,19 @@
 package net.sergeych.lynon
 
-class MemoryBitInput(val packedBits: UByteArray): BitInput() {
-    constructor(bout: MemoryBitOutput): this(bout.toUByteArray())
+class MemoryBitInput(val packedBits: UByteArray,val lastByteBits: Int): BitInput() {
+
+    constructor(bout: MemoryBitOutput): this(bout.toUByteArray(), bout.lastByteBits)
 
     private var index = 0
 
-    override fun getByte(): Int {
-        if( index < packedBits.size ) {
-            return packedBits[index++].toInt()
+    override fun getByte(): DataByte {
+        return if( index < packedBits.size ) {
+            DataByte(
+                packedBits[index++].toInt(),
+                if( index == packedBits.size ) lastByteBits else 8
+            )
         } else {
-            return -1
+            DataByte(-1,0)
         }
     }
 

lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw0.kt

@@ -7,13 +7,11 @@ import kotlin.math.roundToInt
  * LZW compression algorithm: work in progress.
  *
  * Uses Lyng but input/output. Uses automatic code size.
- *
- * TODO: - reset dictionary
  */
 class LZW {
     companion object {
 
-        val MAX_CODE_SIZE = 12
+        val MAX_CODE_SIZE = 17
         val STOP_CODE = (1 shl MAX_CODE_SIZE) - 1
         val MAX_DICT_SIZE = (STOP_CODE * 0.92).roundToInt()
 
@@ -42,7 +40,16 @@ class LZW {
                 } else {
                     val size = sizeInBits(dictionary.size)
                     bitOutput.putBits(dictionary[current]!!,size)
-                    dictionary[combined] = nextCode++
+                    if( dictionary.size >= MAX_DICT_SIZE ) {
+                        bitOutput.putBits(STOP_CODE,size)
+                        dictionary.clear()
+                        nextCode = 256
+                        for (i in 0..255) {
+                            dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
+                        }
+                    }
+                    else
+                        dictionary[combined] = nextCode++
                     current = ByteChunk(ubyteArrayOf(char))
                 }
             }
@@ -72,18 +79,29 @@ class LZW {
             while( !compressed.isEndOfStream ) {
                 val codeSize = sizeInBits(nextCode + 1)
                 val code = compressed.getBitsOrNull(codeSize)?.toInt() ?: break
-                val current = if ( code in dictionary) {
-                    dictionary[code]!!
-                } else if (code == nextCode) {
-                    // Special case for pattern like cScSc
-                    previous + previous[0]
-                } else {
-                    throw IllegalArgumentException("Invalid compressed code: $code")
-                }
 
-                result += current
-                dictionary[nextCode++] = previous + current[0]
-                previous = current
+                if( code == STOP_CODE ) {
+                    nextCode = 256
+                    dictionary.clear()
+                    for (i in 0..255)
+                        dictionary[i] = ubyteArrayOf(i.toUByte())
+                    previous = dictionary[compressed.getBits(9).toInt()]!!
+                }
+                else {
+
+                    val current = if (code in dictionary) {
+                        dictionary[code]!!
+                    } else if (code == nextCode) {
+                        // Special case for pattern like cScSc
+                        previous + previous[0]
+                    } else {
+                        throw IllegalArgumentException("Invalid compressed code: $code")
+                    }
+
+                    result += current
+                    dictionary[nextCode++] = previous + current[0]
+                    previous = current
+                }
             }
 
             return result.toTypedArray().toUByteArray()

lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt

@@ -5,10 +5,11 @@ package net.sergeych.lynon
  */
 class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput(), settings: LynonSettings = LynonSettings.default)
     : LynonEncoder(bout, settings) {
-    fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
 }
 
 /**
  * Variant of [LynonDecoder] that reads from a given `source` using [MemoryBitInput]
  */
-class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
+class LynonUnpacker(source: BitInput) : LynonDecoder(source) {
+    constructor(packer: LynonPacker) : this(MemoryBitInput(packer.bout as MemoryBitOutput))
+}