refs #35 started user class serialization; started MP compression
This commit is contained in:
parent
77f9191387
commit
23dafff453
@ -1112,6 +1112,7 @@ class Compiler(
|
|||||||
// inheritance must alter this code:
|
// inheritance must alter this code:
|
||||||
val newClass = ObjClass(className).apply {
|
val newClass = ObjClass(className).apply {
|
||||||
instanceConstructor = constructorCode
|
instanceConstructor = constructorCode
|
||||||
|
constructorMeta = constructorArgsDeclaration
|
||||||
}
|
}
|
||||||
|
|
||||||
return statement {
|
return statement {
|
||||||
|
@ -10,6 +10,7 @@ open class ObjClass(
|
|||||||
vararg parents: ObjClass,
|
vararg parents: ObjClass,
|
||||||
) : Obj() {
|
) : Obj() {
|
||||||
|
|
||||||
|
var constructorMeta: ArgsDeclaration? = null
|
||||||
var instanceConstructor: Statement? = null
|
var instanceConstructor: Statement? = null
|
||||||
|
|
||||||
val allParentsSet: Set<ObjClass> =
|
val allParentsSet: Set<ObjClass> =
|
||||||
|
@ -2,6 +2,7 @@ package net.sergeych.lyng.obj
|
|||||||
|
|
||||||
import net.sergeych.lyng.Arguments
|
import net.sergeych.lyng.Arguments
|
||||||
import net.sergeych.lyng.Scope
|
import net.sergeych.lyng.Scope
|
||||||
|
import net.sergeych.lynon.LynonEncoder
|
||||||
|
|
||||||
class ObjInstance(override val objClass: ObjClass) : Obj() {
|
class ObjInstance(override val objClass: ObjClass) : Obj() {
|
||||||
|
|
||||||
@ -44,6 +45,18 @@ class ObjInstance(override val objClass: ObjClass) : Obj() {
|
|||||||
return "${objClass.className}($fields)"
|
return "${objClass.className}($fields)"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun serialize(scope: Scope, encoder: LynonEncoder) {
|
||||||
|
val meta = objClass.constructorMeta
|
||||||
|
?: scope.raiseError("can't serialize non-serializable object (no constructor meta)")
|
||||||
|
for( p in meta.params) {
|
||||||
|
val r = readField(scope, p.name)
|
||||||
|
println("serialize ${p.name}=${r.value}")
|
||||||
|
TODO()
|
||||||
|
// encoder.encodeObj(scope, r.value)
|
||||||
|
}
|
||||||
|
// todo: possible vars?
|
||||||
|
}
|
||||||
|
|
||||||
override suspend fun compareTo(scope: Scope, other: Obj): Int {
|
override suspend fun compareTo(scope: Scope, other: Obj): Int {
|
||||||
if (other !is ObjInstance) return -1
|
if (other !is ObjInstance) return -1
|
||||||
if (other.objClass != objClass) return -1
|
if (other.objClass != objClass) return -1
|
||||||
|
@ -4,16 +4,15 @@ import net.sergeych.lyng.Scope
|
|||||||
import net.sergeych.lyng.obj.Obj
|
import net.sergeych.lyng.obj.Obj
|
||||||
import net.sergeych.lyng.obj.ObjClass
|
import net.sergeych.lyng.obj.ObjClass
|
||||||
|
|
||||||
open class LynonDecoder(val bin: BitInput) {
|
open class LynonDecoder(val bin: BitInput,val settings: LynonSettings = LynonSettings.default) {
|
||||||
|
|
||||||
val cache = mutableListOf<Obj>()
|
val cache = mutableListOf<Obj>()
|
||||||
|
|
||||||
inline fun decodeCached(f: LynonDecoder.() -> Obj): Obj {
|
inline fun decodeCached(f: LynonDecoder.() -> Obj): Obj {
|
||||||
return if( bin.getBit() == 0 ) {
|
return if( bin.getBit() == 0 ) {
|
||||||
// unpack and cache
|
// unpack and cache
|
||||||
val cached = bin.getBool()
|
|
||||||
f().also {
|
f().also {
|
||||||
if( cached ) cache.add(it)
|
if( settings.shouldCache(it) ) cache.add(it)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
|
@ -2,28 +2,12 @@ package net.sergeych.lynon
|
|||||||
|
|
||||||
import net.sergeych.lyng.Scope
|
import net.sergeych.lyng.Scope
|
||||||
import net.sergeych.lyng.obj.Obj
|
import net.sergeych.lyng.obj.Obj
|
||||||
import net.sergeych.lyng.obj.ObjBool
|
|
||||||
import net.sergeych.lyng.obj.ObjChar
|
|
||||||
import net.sergeych.lyng.obj.ObjInt
|
|
||||||
|
|
||||||
class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput()) : LynonEncoder(bout) {
|
open class LynonEncoder(val bout: BitOutput,val settings: LynonSettings = LynonSettings.default) {
|
||||||
fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
|
|
||||||
}
|
|
||||||
|
|
||||||
class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
|
|
||||||
|
|
||||||
open class LynonEncoder(val bout: BitOutput) {
|
|
||||||
|
|
||||||
fun shouldCache(obj: Obj): Boolean = when (obj) {
|
|
||||||
is ObjChar -> false
|
|
||||||
is ObjInt -> obj.value > 0x10000FF
|
|
||||||
is ObjBool -> false
|
|
||||||
else -> true
|
|
||||||
}
|
|
||||||
|
|
||||||
val cache = mutableMapOf<Any, Int>()
|
val cache = mutableMapOf<Any, Int>()
|
||||||
|
|
||||||
inline fun encodeCached(item: Any, packer: LynonEncoder.() -> Unit) {
|
private inline fun encodeCached(item: Any, packer: LynonEncoder.() -> Unit) {
|
||||||
if (item is Obj) {
|
if (item is Obj) {
|
||||||
cache[item]?.let { cacheId ->
|
cache[item]?.let { cacheId ->
|
||||||
val size = sizeInBits(cache.size)
|
val size = sizeInBits(cache.size)
|
||||||
@ -31,11 +15,8 @@ open class LynonEncoder(val bout: BitOutput) {
|
|||||||
bout.putBits(cacheId.toULong(), size)
|
bout.putBits(cacheId.toULong(), size)
|
||||||
} ?: run {
|
} ?: run {
|
||||||
bout.putBit(0)
|
bout.putBit(0)
|
||||||
if (shouldCache(item)) {
|
if (settings.shouldCache(item))
|
||||||
bout.putBit(1)
|
|
||||||
cache[item] = cache.size
|
cache[item] = cache.size
|
||||||
} else
|
|
||||||
bout.putBit(0)
|
|
||||||
packer()
|
packer()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
package net.sergeych.lynon
|
||||||
|
|
||||||
|
import net.sergeych.lyng.obj.Obj
|
||||||
|
import net.sergeych.lyng.obj.ObjBool
|
||||||
|
import net.sergeych.lyng.obj.ObjChar
|
||||||
|
import net.sergeych.lyng.obj.ObjInt
|
||||||
|
|
||||||
|
open class LynonSettings() {
|
||||||
|
|
||||||
|
open fun shouldCache(obj: Obj): Boolean = when (obj) {
|
||||||
|
is ObjChar -> false
|
||||||
|
is ObjInt -> obj.value > 0x10000FF
|
||||||
|
is ObjBool -> false
|
||||||
|
else -> true
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
val default = LynonSettings()
|
||||||
|
}
|
||||||
|
}
|
96
lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw0.kt
Normal file
96
lynglib/src/commonMain/kotlin/net/sergeych/lynon/lzw0.kt
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
package net.sergeych.lynon
|
||||||
|
|
||||||
|
import net.sergeych.bintools.ByteChunk
|
||||||
|
import kotlin.math.roundToInt
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LZW compression algorithm: work in progress.
|
||||||
|
*
|
||||||
|
* Uses Lyng but input/output. Uses automatic code size.
|
||||||
|
*
|
||||||
|
* TODO: - reset dictionary
|
||||||
|
*/
|
||||||
|
class LZW {
|
||||||
|
companion object {
|
||||||
|
|
||||||
|
val MAX_CODE_SIZE = 12
|
||||||
|
val STOP_CODE = (1 shl MAX_CODE_SIZE) - 1
|
||||||
|
val MAX_DICT_SIZE = (STOP_CODE * 0.92).roundToInt()
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compresses the input string using LZW algorithm
|
||||||
|
* @param input The string to compress
|
||||||
|
* @return List of compressed codes
|
||||||
|
*/
|
||||||
|
fun compress(input: UByteArray,bitOutput: BitOutput) {
|
||||||
|
// Initialize dictionary with all possible single characters
|
||||||
|
val dictionary = mutableMapOf<ByteChunk, Int>()
|
||||||
|
for (i in 0..255) {
|
||||||
|
// 23
|
||||||
|
dictionary[ByteChunk(ubyteArrayOf(i.toUByte()))] = i
|
||||||
|
}
|
||||||
|
|
||||||
|
var nextCode = 256
|
||||||
|
var current = ByteChunk(ubyteArrayOf())
|
||||||
|
// val result = mutableListOf<Int>()
|
||||||
|
|
||||||
|
for (char in input) {
|
||||||
|
val combined = current + char
|
||||||
|
if (dictionary.containsKey(combined)) {
|
||||||
|
current = combined
|
||||||
|
} else {
|
||||||
|
val size = sizeInBits(dictionary.size)
|
||||||
|
bitOutput.putBits(dictionary[current]!!,size)
|
||||||
|
dictionary[combined] = nextCode++
|
||||||
|
current = ByteChunk(ubyteArrayOf(char))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current.size > 0) {
|
||||||
|
val size = sizeInBits(dictionary.size)
|
||||||
|
bitOutput.putBits(dictionary[current]!!,size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decompresses a list of LZW codes back to the original string
|
||||||
|
* @param compressed The list of compressed codes
|
||||||
|
* @return The decompressed string
|
||||||
|
*/
|
||||||
|
fun decompress(compressed: BitInput): UByteArray {
|
||||||
|
// Initialize dictionary with all possible single characters
|
||||||
|
val dictionary = mutableMapOf<Int, UByteArray>()
|
||||||
|
for (i in 0..255) {
|
||||||
|
dictionary[i] = ubyteArrayOf(i.toUByte())
|
||||||
|
}
|
||||||
|
|
||||||
|
var nextCode = 256
|
||||||
|
var previous = dictionary[compressed.getBits(9).toInt()]!!
|
||||||
|
val result = mutableListOf<UByte>()
|
||||||
|
|
||||||
|
while( !compressed.isEndOfStream ) {
|
||||||
|
val codeSize = sizeInBits(nextCode + 1)
|
||||||
|
val code = compressed.getBitsOrNull(codeSize)?.toInt() ?: break
|
||||||
|
val current = if ( code in dictionary) {
|
||||||
|
dictionary[code]!!
|
||||||
|
} else if (code == nextCode) {
|
||||||
|
// Special case for pattern like cScSc
|
||||||
|
previous + previous[0]
|
||||||
|
} else {
|
||||||
|
throw IllegalArgumentException("Invalid compressed code: $code")
|
||||||
|
}
|
||||||
|
|
||||||
|
result += current
|
||||||
|
dictionary[nextCode++] = previous + current[0]
|
||||||
|
previous = current
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.toTypedArray().toUByteArray()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private operator fun ByteChunk.plus(byte: UByte): ByteChunk {
|
||||||
|
return ByteChunk(data + byte)
|
||||||
|
}
|
14
lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt
Normal file
14
lynglib/src/commonMain/kotlin/net/sergeych/lynon/tools.kt
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
package net.sergeych.lynon
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Variant of [LynonEncoder] that writes to embedded [MemoryBitOutput]
|
||||||
|
*/
|
||||||
|
class LynonPacker(bout: MemoryBitOutput = MemoryBitOutput(), settings: LynonSettings = LynonSettings.default)
|
||||||
|
: LynonEncoder(bout, settings) {
|
||||||
|
fun toUByteArray(): UByteArray = (bout as MemoryBitOutput).toUByteArray()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Variant of [LynonDecoder] that reads from a given `source` using [MemoryBitInput]
|
||||||
|
*/
|
||||||
|
class LynonUnpacker(source: UByteArray) : LynonDecoder(MemoryBitInput(source))
|
@ -3,6 +3,8 @@ import kotlinx.coroutines.test.runTest
|
|||||||
import net.sergeych.lyng.Scope
|
import net.sergeych.lyng.Scope
|
||||||
import net.sergeych.lyng.obj.*
|
import net.sergeych.lyng.obj.*
|
||||||
import net.sergeych.lynon.*
|
import net.sergeych.lynon.*
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
|
|
||||||
class LynonTests {
|
class LynonTests {
|
||||||
@ -189,4 +191,27 @@ class LynonTests {
|
|||||||
assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
|
assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
|
||||||
assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
|
assertEquals(ObjInt(Long.MAX_VALUE), decoder.unpackObject(scope, ObjInt.type))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun testLzw() {
|
||||||
|
// Example usage
|
||||||
|
// val original = "TOBEORNOTTOBEORTOBEORNOT"
|
||||||
|
val original = Files.readString(Path.of("../sample_texts/dikkens_hard_times.txt"))
|
||||||
|
// println("Original: $original")
|
||||||
|
println("Length: ${original.length}")
|
||||||
|
|
||||||
|
// Compress
|
||||||
|
val out = MemoryBitOutput()
|
||||||
|
LZW.compress(original.encodeToByteArray().toUByteArray(), out)
|
||||||
|
// println("\nCompressed codes: ${out.toUByteArray().toDump()}")
|
||||||
|
println("Number of codes: ${out.toUByteArray().size}")
|
||||||
|
|
||||||
|
// // Decompress
|
||||||
|
val decompressed = LZW.decompress(MemoryBitInput(out.toUByteArray())).toByteArray().decodeToString()
|
||||||
|
// println("\nDecompressed: $decompressed")
|
||||||
|
println("Length: ${decompressed.length}")
|
||||||
|
|
||||||
|
// Verification
|
||||||
|
println("\nOriginal and decompressed match: ${original == decompressed}")
|
||||||
|
}
|
||||||
}
|
}
|
2251
sample_texts/dikkens_hard_times.txt
Normal file
2251
sample_texts/dikkens_hard_times.txt
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user