fix #33 minimal Buffer with docs, in a separate package

This commit is contained in:
Sergey Chernov 2025-07-09 17:55:29 +03:00
parent 26282d3e22
commit 23006b5caa
17 changed files with 420 additions and 60 deletions

110
docs/Buffer.md Normal file
View File

@ -0,0 +1,110 @@
# Binary `Buffer`
Buffers are effective unsigned byte arrays of fixed size. Buffers content is mutable,
unlike its size. Buffers are comparable and implement [Array], thus [Collection] and [Iterable]. Buffer iterators return its contents as unsigned bytes converted to `Int`
Buffers needs to be imported with `import lyng.buffer`:
import lyng.buffer
assertEquals(5, Buffer("Hello").size)
>>> void
## Constructing
There are a lo of ways to construct a buffer:
import lyng.buffer
// from string using utf8 encoding:
assertEquals( 5, Buffer("hello").size )
// from bytes, e.g. integers in range 0..255
assertEquals( 255, Buffer(1,2,3,255).last() )
// from whatever iterable that produces bytes, e.g.
// integers in 0..255 range:
assertEquals( 129, Buffer([1,2,129]).last() )
// Empty buffer of fixed size:
assertEquals(100, Buffer(100).size)
assertEquals(0, Buffer(100)[0])
// Note that you can use list iteral to create buffer with 1 byte:
assertEquals(1, Buffer([100]).size)
assertEquals(100, Buffer([100])[0])
>>> void
## Accessing an modifying
Buffer implement [Array] and therefore can be accessed and modified with indexing:
import lyng.buffer
val b1 = Buffer( 1, 2, 3)
assertEquals( 2, b1[1] )
b1[0] = 199
assertEquals(199, b1[0])
>>> void
Buffer provides concatenation with another Buffer:
import lyng.buffer
val b = Buffer(101, 102)
assertEquals( Buffer(101, 102, 1, 2), b + [1,2])
>>> void
## Comparing
Buffers are comparable with other buffers:
import lyng.buffer
val b1 = Buffer(1, 2, 3)
val b2 = Buffer(1, 2, 3)
val b3 = Buffer(b2)
b3[0] = 101
assert( b3 > b1 )
assert( b2 == b1 )
// longer string with same prefix is considered bigger:
assert( b2 + "!".characters() > b1 )
// note that characters() provide Iterable of characters that
// can be concatenated to Buffer
>>> void
## Slicing
As with [List], it is possible to use ranges as indexes to slice a Buffer:
import lyng.buffer
val a = Buffer( 100, 101, 102, 103, 104, 105 )
assertEquals( a[ 0..1 ], Buffer(100, 101) )
assertEquals( a[ 0 ..< 2 ], Buffer(100, 101) )
assertEquals( a[ ..< 2 ], Buffer(100, 101) )
assertEquals( a[ 4.. ], Buffer(104, 105) )
assertEquals( a[ 2..3 ], Buffer(102, 103) )
>>> void
## Members
| name | meaning | type |
|---------------------|--------------------------------------|-------|
| `size` | size | Int |
| `+=` | add one or more elements | Any |
| `+`, `union` | union sets | Any |
| `-`, `subtract` | subtract sets | Any |
| `*`, `intersect` | subtract sets | Any |
| `remove(items...)` | remove one or more items | Range |
| `contains(element)` | check the element is in the list (1) | |
(1)
: optimized implementation that override `Iterable` one
Also, it inherits methods from [Iterable].
[Range]: Range.md

View File

@ -3,6 +3,8 @@
Map is a mutable collection of key-value pars, where keys are unique. Maps could be created with
constructor or `.toMap` methods. When constructing from a list, each list item must be a [Collection] with exactly 2 elements, for example, a [List].
Important thing is that maps can't contain `null`: it is used to return from missing elements.
Constructed map instance is of class `Map` and implements `Collection` (and therefore `Iterable`)
val map = Map( ["foo", 1], ["bar", "buzz"] )
@ -16,7 +18,7 @@ Map keys could be any objects (hashable, e.g. with reasonable hashCode, most of
val map = Map( ["foo", 1], ["bar", "buzz"], [42, "answer"] )
assert( map["bar"] == "buzz")
assert( map[42] == "answer" )
assertThrows { map["nonexistent"] }
assertEquals( null, map["nonexisting"])
assert( map.getOrNull(101) == null )
assert( map.getOrPut(911) { "nine-eleven" } == "nine-eleven" )
// now 91 entry is set:

View File

@ -530,7 +530,8 @@ The simplest way to concatenate lists is using `+` and `+=`:
list += [2, 1]
// or you can append a single element:
list += "end"
assert( list == [1, 2, 2, 1, "end"])
assertEquals( list, [1, 2, 2, 1, "end"])
void
>>> void
***Important note***: the pitfall of using `+=` is that you can't append in [Iterable] instance as an object: it will always add all its contents. Use `list.add` to add a single iterable instance:
@ -641,6 +642,11 @@ You can get ranges to extract a portion from a list:
assertEquals( [2,3], list[1..<3])
>>> void
# Buffers
[Buffer] is a special implementation of an [Array] of unsigned bytes, in the
[separate file](Buffer.md).
# Sets
Set are unordered collection of unique elements, see [Set]. Sets are [Iterable] but have no indexing access.
@ -740,13 +746,13 @@ You can thest that _when expression_ is _contained_, or not contained, in some o
Typical builtin types that are containers (e.g. support `conain`):
| class | notes |
|------------|--------------------------------------------|
| Collection | contains an element (1) |
| Array | faster maybe that Collection's |
| List | faster than Array's |
| String | character in string or substring in string |
| Range | object is included in the range (2) |
| class | notes |
|------------|------------------------------------------------|
| Collection | contains an element (1) |
| Array | faster maybe that Collection's |
| List | faster than Array's |
| String | character in string or substring in string (3) |
| Range | object is included in the range (2) |
(1)
: Iterable is not the container as it can be infinite
@ -760,6 +766,9 @@ Typical builtin types that are containers (e.g. support `conain`):
assert( "x" !in 'a'..'z') // string in character range: could be error
>>> void
(3)
: `String` also can provide array of characters directly with `str.characters()`, which is [Iterable] and [Array]. String itself is not iterable as otherwise it will interfere when adding strigns to lists (it will add _characters_ it it would be iterable).
So we recommend not to mix characters and string ranges; use `ch in str` that works
as expected:
@ -1216,9 +1225,10 @@ Typical set of String functions includes:
| s1 += s2 | self-modifying concatenation |
| toReal() | attempts to parse string as a Real value |
| toInt() | parse string to Int value |
| | |
| characters() | create [List] of characters (1) |
(1)
: List is mutable therefore a new copy is created on each call.
@ -1256,4 +1266,5 @@ See [math functions](math.md). Other general purpose functions are:
[String]: String.md
[string formatting]: https://github.com/sergeych/mp_stools?tab=readme-ov-file#sprintf-syntax-summary
[Set]: Set.md
[Map]: Map.md
[Map]: Map.md
[Buffer]: Buffer.md

View File

@ -4,7 +4,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
group = "net.sergeych"
version = "0.7.2-SNAPSHOT"
version = "0.7.3-SNAPSHOT"
buildscript {
repositories {
@ -20,7 +20,7 @@ plugins {
alias(libs.plugins.kotlinMultiplatform)
alias(libs.plugins.androidLibrary)
// alias(libs.plugins.vanniktech.mavenPublish)
kotlin("plugin.serialization") version "2.1.20"
kotlin("plugin.serialization") version "2.2.0"
id("com.codingfeline.buildkonfig") version "0.17.1"
`maven-publish`
}

View File

@ -274,9 +274,7 @@ class Compiler(
if (x == ObjNull && isOptional) ObjNull.asReadonly
else x.getAt(cxt, i).asMutable
}) { cxt, newValue ->
val i = (index.execute(cxt) as? ObjInt)?.value?.toInt()
?: cxt.raiseError("index must be integer")
left.getter(cxt).value.putAt(cxt, i, newValue)
left.getter(cxt).value.putAt(cxt, index.execute(cxt), newValue)
}
} ?: run {
// array literal

View File

@ -165,7 +165,6 @@ class CompilerContext(val tokens: List<Token>) {
*/
fun skipWsTokens(): Token {
while( current().type in wstokens ) {
println("skipws ${current()}")
next()
}
return next()

View File

@ -220,7 +220,7 @@ open class Obj {
suspend fun getAt(scope: Scope, index: Int): Obj = getAt(scope, ObjInt(index.toLong()))
open suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
open suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
scope.raiseNotImplemented("indexing")
}
@ -280,7 +280,18 @@ open class Obj {
args.firstAndOnly().callOn(copy(Arguments(thisObj)))
thisObj
}
addFn("getAt") {
requireExactCount(1)
thisObj.getAt(this, requiredArg<Obj>(0))
}
addFn("putAt") {
requireExactCount(2)
val newValue = args[1]
thisObj.putAt(this, requiredArg<Obj>(0), newValue)
newValue
}
}
inline fun from(obj: Any?): Obj {
@ -349,7 +360,7 @@ object ObjNull : Obj() {
scope.raiseNPE()
}
override suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
scope.raiseNPE()
}

View File

@ -0,0 +1,145 @@
package net.sergeych.lyng
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.toList
import kotlin.math.min
class ObjBuffer(val byteArray: UByteArray) : Obj() {
override val objClass: ObjClass = type
fun checkIndex(scope: Scope, index: Obj): Int {
if (index !is ObjInt)
scope.raiseIllegalArgument("index must be Int")
val i = index.value.toInt()
if (i < 0) scope.raiseIllegalArgument("index must be positive")
if (i >= byteArray.size)
scope.raiseIndexOutOfBounds("index $i is out of bounds 0..<${byteArray.size}")
return i
}
override suspend fun getAt(scope: Scope, index: Obj): Obj {
// notice: we create a copy if content, so we don't want it
// to be treated as modifiable, or putAt will not be called:
return if (index is ObjRange) {
val start: Int = index.startInt(scope)
val end: Int = index.exclusiveIntEnd(scope) ?: size
ObjBuffer(byteArray.sliceArray(start..<end))
}
else ObjInt(byteArray[checkIndex(scope, index)].toLong(), true)
}
override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
byteArray[checkIndex(scope, index.toObj())] = when (newValue) {
is ObjInt -> newValue.value.toUByte()
is ObjChar -> newValue.value.code.toUByte()
else -> scope.raiseIllegalArgument(
"invalid byte value for buffer at index ${index.inspect()}: ${newValue.inspect()}"
)
}
}
val size by byteArray::size
override suspend fun compareTo(scope: Scope, other: Obj): Int {
if (other !is ObjBuffer) return -1
val limit = min(size, other.size)
for (i in 0..<limit) {
val own = byteArray[i]
val their = other.byteArray[i]
if (own < their) return -1
else if (own > their) return 1
}
if (size < other.size) return -1
if (size > other.size) return 1
return 0
}
override suspend fun plus(scope: Scope, other: Obj): Obj {
return if( other is ObjBuffer)
ObjBuffer(byteArray + other.byteArray)
else if( other.isInstanceOf(ObjIterable)) {
ObjBuffer(
byteArray + other.toFlow(scope).map { it.toLong().toUByte() }.toList().toTypedArray().toUByteArray()
)
} else scope.raiseIllegalArgument("can't concatenate buffer with ${other.inspect()}")
}
override fun toString(): String {
return "Buffer(${byteArray.toList()})"
}
companion object {
private suspend fun createBufferFrom(scope: Scope, obj: Obj): ObjBuffer =
when (obj) {
is ObjBuffer -> ObjBuffer(obj.byteArray.copyOf())
is ObjInt -> {
if (obj.value < 0)
scope.raiseIllegalArgument("buffer size must be positive")
val data = UByteArray(obj.value.toInt())
ObjBuffer(data)
}
is ObjString -> ObjBuffer(obj.value.encodeToByteArray().asUByteArray())
else -> {
if (obj.isInstanceOf(ObjIterable)) {
ObjBuffer(
obj.toFlow(scope).map { it.toLong().toUByte() }.toList().toTypedArray().toUByteArray()
)
} else
scope.raiseIllegalArgument(
"can't construct buffer from ${obj.inspect()}"
)
}
}
val type = object : ObjClass("Buffer", ObjArray) {
override suspend fun callOn(scope: Scope): Obj {
val args = scope.args.list
return when (args.size) {
// empty buffer
0 -> ObjBuffer(ubyteArrayOf())
1 -> createBufferFrom(scope, args[0])
else -> {
// create buffer from array, each argument should be a byte then:
val data = UByteArray(args.size)
for ((i, b) in args.withIndex()) {
val code = when (b) {
is ObjChar -> b.value.code.toUByte()
is ObjInt -> b.value.toUByte()
else -> scope.raiseIllegalArgument(
"invalid byte value for buffer constructor at index $i: ${b.inspect()}"
)
}
data[i] = code
}
ObjBuffer(data)
}
}
}
}.apply {
createField("size",
statement {
(thisObj as ObjBuffer).byteArray.size.toObj()
}
)
addFn("decodeUtf8") {
ObjString(
thisAs<ObjBuffer>().byteArray.toByteArray().decodeToString()
)
}
// )
// addFn("getAt") {
// requireExactCount(1)
// thisAs<ObjList>().getAt(this, requiredArg<Obj>(0))
// }
// addFn("putAt") {
// requireExactCount(2)
// val newValue = args[1]
// thisAs<ObjList>().putAt(this, requiredArg<ObjInt>(0).value.toInt(), newValue)
// newValue
// }
}
}
}

View File

@ -1,6 +1,6 @@
package net.sergeych.lyng
data class ObjInt(var value: Long) : Obj(), Numeric {
class ObjInt(var value: Long,val isConst: Boolean = false) : Obj(), Numeric {
override val asStr get() = ObjString(value.toString())
override val longValue get() = value
override val doubleValue get() = value.toDouble()
@ -70,7 +70,7 @@ data class ObjInt(var value: Long) : Obj(), Numeric {
* assignment
*/
override suspend fun assign(scope: Scope, other: Obj): Obj? {
return if (other is ObjInt) {
return if (!isConst && other is ObjInt) {
value = other.value
this
} else null
@ -90,8 +90,8 @@ data class ObjInt(var value: Long) : Obj(), Numeric {
}
companion object {
val Zero = ObjInt(0)
val One = ObjInt(1)
val Zero = ObjInt(0, true)
val One = ObjInt(1, true)
val type = ObjClass("Int")
}
}

View File

@ -2,11 +2,6 @@ package net.sergeych.lyng
class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
init {
for (p in objClass.parents)
parentInstances.add(p.defaultInstance())
}
override fun toString(): String = "[${
list.joinToString(separator = ", ") { it.inspect() }
}]"
@ -51,9 +46,8 @@ class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
}
}
override suspend fun putAt(scope: Scope, index: Int, newValue: Obj) {
val i = index
list[i] = newValue
override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
list[index.toInt()] = newValue
}
override suspend fun compareTo(scope: Scope, other: Obj): Int {
@ -136,16 +130,6 @@ class ObjList(val list: MutableList<Obj> = mutableListOf()) : Obj() {
(thisObj as ObjList).list.size.toObj()
}
)
addFn("getAt") {
requireExactCount(1)
thisAs<ObjList>().getAt(this, requiredArg<Obj>(0))
}
addFn("putAt") {
requireExactCount(2)
val newValue = args[1]
thisAs<ObjList>().putAt(this, requiredArg<ObjInt>(0).value.toInt(), newValue)
newValue
}
createField("add",
statement {
val l = thisAs<ObjList>().list

View File

@ -39,7 +39,11 @@ class ObjMap(val map: MutableMap<Obj, Obj> = mutableMapOf()) : Obj() {
override val objClass = type
override suspend fun getAt(scope: Scope, index: Obj): Obj =
map.getOrElse(index) { scope.raiseNoSuchElement() }
map.get(index) ?: ObjNull
override suspend fun putAt(scope: Scope, index: Obj, newValue: Obj) {
map[index] = newValue
}
override suspend fun contains(scope: Scope, other: Obj): Boolean {
return other in map

View File

@ -15,6 +15,31 @@ class ObjRange(val start: Obj?, val end: Obj?, val isEndInclusive: Boolean) : Ob
return result.toString()
}
/**
* IF end is open (null/ObjNull), returns null
* Otherwise, return correct value for the exclusive end
* raises [ObjIllegalArgumentException] if end is not ObjInt
*/
fun exclusiveIntEnd(scope: Scope): Int? =
if (end == null || end is ObjNull) null
else {
if (end !is ObjInt) scope.raiseIllegalArgument("end is not int")
if (isEndInclusive) end.value.toInt() + 1 else end.value.toInt()
}
/**
* If start is null/ObjNull, returns 0
* if start is not ObjInt, raises [ObjIllegalArgumentException]
* otherwise returns start.value.toInt()
*/
fun startInt(scope: Scope): Int =
if( start == null || start is ObjNull ) 0
else {
if( start is ObjInt ) start.value.toInt()
else scope.raiseIllegalArgument("start is not Int: ${start.inspect()}")
}
suspend fun containsRange(scope: Scope, other: ObjRange): Boolean {
if (start != null) {
// our start is not -∞ so other start should be GTE or is not contained:

View File

@ -15,7 +15,6 @@ data class ObjString(val value: String) : Obj() {
// return i
// }
override suspend fun compareTo(scope: Scope, other: Obj): Int {
if (other !is ObjString) return -2
return this.value.compareTo(other.value)
@ -114,6 +113,11 @@ data class ObjString(val value: String) : Obj() {
addFn("upper") {
thisAs<ObjString>().value.uppercase().let(::ObjString)
}
addFn("characters") {
ObjList(
thisAs<ObjString>().value.map { ObjChar(it) }.toMutableList()
)
}
addFn("size") { ObjInt(thisAs<ObjString>().value.length.toLong()) }
addFn("toReal") { ObjReal(thisAs<ObjString>().value.toDouble())}
}

View File

@ -174,7 +174,14 @@ class Script(
}
}
val defaultImportManager: ImportManager by lazy { ImportManager(rootScope, SecurityManager.allowAll) }
val defaultImportManager: ImportManager by lazy {
ImportManager(rootScope, SecurityManager.allowAll).apply {
addPackage("lyng.buffer") {
it.addConst("Buffer", ObjBuffer.type)
}
}
}
}
}

View File

@ -1,8 +1,8 @@
package net.sergeych.lyng.pacman
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import net.sergeych.lyng.*
import net.sergeych.synctools.ProtectedOp
import net.sergeych.synctools.withLock
/**
* Import manager allow to register packages with builder lambdas and act as an
@ -17,7 +17,7 @@ import net.sergeych.lyng.*
class ImportManager(
rootScope: Scope = Script.defaultImportManager.newModule(),
securityManager: SecurityManager = SecurityManager.allowAll
): ImportProvider(rootScope, securityManager) {
) : ImportProvider(rootScope, securityManager) {
private inner class Entry(
val packageName: String,
@ -36,7 +36,7 @@ class ImportManager(
/**
* Inner provider does not lock [access], the only difference; it is meant to be used
* Inner provider does not lock [op], the only difference; it is meant to be used
* exclusively by the coroutine that starts actual import chain
*/
private inner class InternalProvider : ImportProvider(rootScope) {
@ -52,7 +52,9 @@ class ImportManager(
private val imports = mutableMapOf<String, Entry>()
private val access = Mutex()
val op = ProtectedOp()
/**
* Register new package that can be imported. It is not possible to unregister or
@ -65,8 +67,8 @@ class ImportManager(
* @param name package name
* @param builder lambda to create actual package using the given [ModuleScope]
*/
suspend fun addPackage(name: String, builder: suspend (ModuleScope) -> Unit) {
access.withLock {
fun addPackage(name: String, builder: suspend (ModuleScope) -> Unit) {
op.withLock {
if (name in imports)
throw IllegalArgumentException("Package $name already exists")
imports[name] = Entry(name, builder)
@ -77,8 +79,8 @@ class ImportManager(
* Bulk [addPackage] with slightly better performance
*/
@Suppress("unused")
suspend fun addPackages(registrationData: List<Pair<String, suspend (ModuleScope) -> Unit>>) {
access.withLock {
fun addPackages(registrationData: List<Pair<String, suspend (ModuleScope) -> Unit>>) {
op.withLock {
for (pp in registrationData) {
if (pp.first in imports)
throw IllegalArgumentException("Package ${pp.first} already exists")
@ -89,7 +91,7 @@ class ImportManager(
/**
* Perform actual import or return ready scope. __It must only be called when
* [access] is locked__, e.g. only internally
* [op] is locked__, e.g. only internally
*/
private suspend fun doImport(packageName: String, pos: Pos): ModuleScope {
val entry = imports[packageName] ?: throw ImportException(pos, "package not found: $packageName")
@ -102,8 +104,8 @@ class ImportManager(
/**
* Add packages that only need to compile [Source].
*/
suspend fun addSourcePackages(vararg sources: Source) {
for( s in sources) {
fun addSourcePackages(vararg sources: Source) {
for (s in sources) {
addPackage(s.extractPackageName()) {
it.eval(s)
}
@ -114,12 +116,12 @@ class ImportManager(
/**
* Add source packages using package name as [Source.fileName], for simplicity
*/
suspend fun addTextPackages(vararg sourceTexts: String) {
for( s in sourceTexts) {
fun addTextPackages(vararg sourceTexts: String) {
for (s in sourceTexts) {
var source = Source("tmp", s)
val packageName = source.extractPackageName()
source = Source(packageName, s)
addPackage(packageName) { it.eval(source)}
addPackage(packageName) { it.eval(source) }
}
}

View File

@ -2460,4 +2460,57 @@ class ScriptTest {
""".trimIndent())
}
@Test
fun testMaps() = runTest {
eval(
"""
val map = Map( "a" => 1, "b" => 2 )
assertEquals( 1, map["a"] )
assertEquals( 2, map["b"] )
assertEquals( null, map["c"] )
map["c"] = 3
assertEquals( 3, map["c"] )
""".trimIndent()
)
}
@Test
fun testBuffer() = runTest {
eval("""
import lyng.buffer
assertEquals( 0, Buffer().size )
assertEquals( 3, Buffer(1, 2, 3).size )
assertEquals( 5, Buffer("hello").size )
val buffer = Buffer("Hello")
assertEquals( 5, buffer.size)
assertEquals('l'.code, buffer[2] )
assertEquals('l'.code, buffer[3] )
assertEquals("Hello", buffer.decodeUtf8())
buffer[2] = 101
assertEquals(101, buffer[2])
assertEquals("Heelo", buffer.decodeUtf8())
""".trimIndent())
}
@Test
fun testBufferCompare() = runTest {
eval("""
import lyng.buffer
println("Hello".characters())
val b1 = Buffer("Hello")
val b2 = Buffer("Hello".characters())
assertEquals( b1, b2 )
val b3 = b1 + Buffer("!")
assertEquals( "Hello!", b3.decodeUtf8())
assert( b3 > b1 )
""".trimIndent())
}
}

View File

@ -265,6 +265,11 @@ class BookTest {
runDocTests("../docs/Map.md")
}
@Test
fun testBuffer() = runTest {
runDocTests("../docs/Buffer.md")
}
@Test
fun testSampleBooks() = runTest {
for (bt in Files.list(Paths.get("../docs/samples")).toList()) {