Files
lyng/lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt
T

963 lines
33 KiB
Kotlin

/*
* Copyright 2026 Sergey S. Chernov real.sergeych@gmail.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package net.sergeych.lyng
val digitsSet = ('0'..'9').toSet()
val digits = { d: Char -> d in digitsSet }
val hexDigits = digitsSet + ('a'..'f') + ('A'..'F')
val idNextChars = { d: Char -> d.isLetter() || d == '_' || d.isDigit() || d == '$' || d == '~' }
@Suppress("unused")
val idFirstChars = { d: Char -> d.isLetter() || d == '_' || d == '$' }
fun parseLyng(source: Source): List<Token> {
val p = Parser(
fromPos = source.startPos,
interpolationEnabled = detectInterpolationEnabled(source.text)
)
val tokens = mutableListOf<Token>()
do {
val t = p.nextToken()
tokens += t
} while (t.type != Token.Type.EOF)
return tokens
}
private fun parseLyng(source: Source, interpolationEnabled: Boolean): List<Token> {
val p = Parser(
fromPos = source.startPos,
interpolationEnabled = interpolationEnabled
)
val tokens = mutableListOf<Token>()
do {
val t = p.nextToken()
tokens += t
} while (t.type != Token.Type.EOF)
return tokens
}
private fun detectInterpolationEnabled(text: String): Boolean {
// Per-file feature switch in leading comments:
// // feature: interpolation: off
// // feature: interpolation: on
var enabled = true
val lines = text.split('\n')
var i = 0
while (i < lines.size) {
val line = lines[i]
val trimmed = line.trim()
if (trimmed.isBlank()) {
i++
continue
}
if (i == 0 && trimmed.startsWith("#!")) {
i++
continue
}
if (trimmed.startsWith("//")) {
val m = Regex("^//\\s*feature\\s*:\\s*interpolation\\s*:\\s*(on|off)\\s*$", RegexOption.IGNORE_CASE)
.matchEntire(trimmed)
if (m != null) {
enabled = m.groupValues[1].equals("on", ignoreCase = true)
}
i++
continue
}
if (trimmed.startsWith("/*")) {
// Skip leading block comment(s); directives are intentionally line-comment based.
var j = i
var closed = false
while (j < lines.size) {
if (lines[j].contains("*/")) {
closed = true
break
}
j++
}
if (!closed) break
i = j + 1
continue
}
break
}
return enabled
}
private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = true) {
private val pos = MutablePos(fromPos)
private val bufferedTokens = ArrayDeque<Token>()
/**
* Immutable copy of current position
*/
private val currentPos: Pos get() = pos.toPos()
private fun raise(msg: String): Nothing = throw ScriptError(currentPos, msg)
fun nextToken(): Token {
if (bufferedTokens.isNotEmpty()) return bufferedTokens.removeFirst()
skipws()
if (pos.end) return Token("", currentPos, Token.Type.EOF)
val from = currentPos
return when (val ch = pos.currentChar.also { pos.advance() }) {
'(' -> Token("(", from, Token.Type.LPAREN)
')' -> Token(")", from, Token.Type.RPAREN)
'{' -> Token("{", from, Token.Type.LBRACE)
'}' -> Token("}", from, Token.Type.RBRACE)
'[' -> Token("[", from, Token.Type.LBRACKET)
']' -> Token("]", from, Token.Type.RBRACKET)
',' -> Token(",", from, Token.Type.COMMA)
';' -> Token(";", from, Token.Type.SEMICOLON)
'=' -> {
if (pos.currentChar == '=') {
pos.advance()
when (currentChar) {
'=' -> {
pos.advance()
Token("===", from, Token.Type.REF_EQ)
}
else -> Token("==", from, Token.Type.EQ)
}
} else if (currentChar == '>') {
pos.advance()
Token("=>", from, Token.Type.EQARROW)
} else if (currentChar == '~') {
pos.advance()
Token("=~", from, Token.Type.MATCH)
} else
Token("=", from, Token.Type.ASSIGN)
}
'+' -> {
when (currentChar) {
'+' -> {
pos.advance()
Token("++", from, Token.Type.PLUS2)
}
'=' -> {
pos.advance()
Token("+=", from, Token.Type.PLUSASSIGN)
}
else ->
Token("+", from, Token.Type.PLUS)
}
}
'-' -> {
when (currentChar) {
'-' -> {
pos.advance()
Token("--", from, Token.Type.MINUS2)
}
'=' -> {
pos.advance()
Token("-=", from, Token.Type.MINUSASSIGN)
}
'>' -> {
pos.advance()
Token("->", from, Token.Type.ARROW)
}
else -> Token("-", from, Token.Type.MINUS)
}
}
'*' -> {
if (currentChar == '=') {
pos.advance()
Token("*=", from, Token.Type.STARASSIGN)
} else
Token("*", from, Token.Type.STAR)
}
'/' -> when (currentChar) {
'/' -> {
pos.advance()
val body = loadToEndOfLine()
// Include the leading '//' and do not trim; keep exact lexeme (excluding preceding codepoint)
Token("//" + body, from, Token.Type.SINGLE_LINE_COMMENT)
}
'*' -> {
pos.advance()
val content = loadTo("*/")
?: throw ScriptError(from, "Unterminated multiline comment")
// loadTo consumes the closing fragment, so we are already after */
Token("/*" + content + "*/", from, Token.Type.MULTILINE_COMMENT)
}
'=' -> {
pos.advance()
Token("/=", from, Token.Type.SLASHASSIGN)
}
else -> Token("/", from, Token.Type.SLASH)
}
'%' -> when (currentChar) {
'=' -> {
pos.advance(); Token("%=", from, Token.Type.PERCENTASSIGN)
}
else -> Token("%", from, Token.Type.PERCENT)
}
'.' -> {
// could be: dot, range .. or ..<, or ellipsis ...:
if (currentChar == '.') {
pos.advance()
// .. already parsed:
when (currentChar) {
'.' -> {
pos.advance()
Token("...", from, Token.Type.ELLIPSIS)
}
'<' -> {
pos.advance()
Token("..<", from, Token.Type.DOTDOTLT)
}
else -> {
Token("..", from, Token.Type.DOTDOT)
}
}
} else
Token(".", from, Token.Type.DOT)
}
'<' -> {
if (currentChar == '=') {
pos.advance()
if (currentChar == '>') {
pos.advance()
Token("<=>", from, Token.Type.SHUTTLE)
} else {
Token("<=", from, Token.Type.LTE)
}
} else if (currentChar == '<') {
// Shift left <<
pos.advance()
Token("<<", from, Token.Type.SHL)
} else
Token("<", from, Token.Type.LT)
}
'>' -> {
if (currentChar == '=') {
pos.advance()
Token(">=", from, Token.Type.GTE)
} else if (currentChar == '>') {
// Shift right >>
pos.advance()
Token(">>", from, Token.Type.SHR)
} else
Token(">", from, Token.Type.GT)
}
'!' -> {
if (currentChar == 'i') {
// Potentially !in / !is, but only if a word boundary follows
pos.advance()
when (currentChar) {
'n' -> {
pos.advance()
// if next char continues an identifier, it's actually '!'+identifier starting with "in..."
if (idNextChars(currentChar)) {
// backtrack to right after '!'
pos.back()
pos.back()
Token("!", from, Token.Type.NOT)
} else
Token("!in", from, Token.Type.NOTIN)
}
's' -> {
pos.advance()
// if next char continues an identifier, it's actually '!'+identifier starting with "is..."
if (idNextChars(currentChar)) {
// backtrack to right after '!'
pos.back()
pos.back()
Token("!", from, Token.Type.NOT)
} else
Token("!is", from, Token.Type.NOTIS)
}
else -> {
// it was just '!i' followed by something else; revert one step and return '!'
pos.back()
Token("!", from, Token.Type.NOT)
}
}
} else if (currentChar == '=') {
pos.advance()
if (currentChar == '=') {
pos.advance()
Token("!==", from, Token.Type.REF_NEQ)
} else
Token("!=", from, Token.Type.NEQ)
} else if (currentChar == '~') {
pos.advance()
Token("!~", from, Token.Type.NOTMATCH)
} else
Token("!", from, Token.Type.NOT)
}
'|' -> {
if (currentChar == '|') {
pos.advance()
Token("||", from, Token.Type.OR)
} else
Token("|", from, Token.Type.BITOR)
}
'&' -> {
if (currentChar == '&') {
pos.advance()
Token("&&", from, Token.Type.AND)
} else
Token("&", from, Token.Type.BITAND)
}
'^' -> Token("^", from, Token.Type.BITXOR)
'~' -> Token("~", from, Token.Type.BITNOT)
'@' -> {
val label = loadChars(idNextChars)
if (label.isNotEmpty()) Token(label, from, Token.Type.ATLABEL)
else raise("unexpected @ character")
}
'\n' -> Token("\n", from, Token.Type.NEWLINE)
':' -> {
if (currentChar == ':') {
pos.advance()
Token("::", from, Token.Type.COLONCOLON)
} else
Token(":", from, Token.Type.COLON)
}
'"' -> loadStringTokens(from)
in digitsSet -> {
pos.back()
decodeNumber(loadChars { it in digitsSet || it == '_' }, from)
}
'\'' -> {
val start = pos.toPos()
var value = currentChar
if (currentChar == '\\') {
pos.advance()
if (pos.end) throw ScriptError(start, "unterminated character literal")
value = when (currentChar) {
'n' -> {
pos.advance()
'\n'
}
'r' -> {
pos.advance()
'\r'
}
't' -> {
pos.advance()
'\t'
}
'\'' -> {
pos.advance()
'\''
}
'\\' -> {
pos.advance()
'\\'
}
'u' -> loadUnicodeEscape(start)
else -> throw ScriptError(currentPos, "unsupported escape character: $currentChar")
}
} else {
pos.advance()
}
if (currentChar != '\'') throw ScriptError(currentPos, "expected end of character literal: '")
pos.advance()
Token(value.toString(), start, Token.Type.CHAR)
}
'?' -> {
when (currentChar) {
'=' -> { pos.advance(); Token("?=", from, Token.Type.IFNULLASSIGN) }
':' -> { pos.advance(); Token("?:", from, Token.Type.ELVIS) }
'?' -> { pos.advance(); Token("??", from, Token.Type.ELVIS) }
'.' -> { pos.advance(); Token("?.", from, Token.Type.NULL_COALESCE) }
'[' -> { pos.advance(); Token("?[", from, Token.Type.NULL_COALESCE_INDEX) }
'(' -> { pos.advance(); Token("?(", from, Token.Type.NULL_COALESCE_INVOKE) }
'{' -> { pos.advance(); Token("?{", from, Token.Type.NULL_COALESCE_BLOCKINVOKE) }
else -> {
Token("?", from, Token.Type.QUESTION)
}
}
}
else -> {
// text infix operators:
// Labels processing is complicated!
// some@ statement: label 'some', ID 'statement'
// statement@some: ID 'statement', LABEL 'some'!
if (idNextChars(ch)) {
val text = ch + loadChars(idNextChars)
if (currentChar == '@') {
pos.advance()
if (currentChar.isLetter()) {
// break@label or like
pos.back()
Token(text, from, Token.Type.ID)
} else
Token(text, from, Token.Type.LABEL)
} else
when (text) {
"in" -> Token("in", from, Token.Type.IN)
"is" -> Token("is", from, Token.Type.IS)
"by" -> Token("by", from, Token.Type.BY)
"step" -> Token("step", from, Token.Type.STEP)
"object" -> Token("object", from, Token.Type.OBJECT)
"as" -> {
// support both `as` and tight `as?` without spaces
if (currentChar == '?') { pos.advance(); Token("as?", from, Token.Type.ASNULL) }
else Token("as", from, Token.Type.AS)
}
else -> Token(text, from, Token.Type.ID)
}
} else
raise("can't parse token")
}
}
}
private fun decodeNumber(p1: String, start: Pos): Token =
if (pos.end)
Token(p1, start, Token.Type.INT)
else if (currentChar == 'e' || currentChar == 'E') {
pos.advance()
var negative = false
if (currentChar == '+')
pos.advance()
else if (currentChar == '-') {
negative = true
pos.advance()
}
var p3 = loadChars(digits)
if (negative) p3 = "-$p3"
Token("${p1}e$p3", start, Token.Type.REAL)
} else if (currentChar == '.') {
// could be decimal
pos.advance()
if (currentChar in digitsSet) {
// decimal part
val p2 = loadChars(digits)
// with exponent?
if (currentChar == 'e' || currentChar == 'E') {
pos.advance()
var negative = false
if (currentChar == '+')
pos.advance()
else if (currentChar == '-') {
negative = true
pos.advance()
}
var p3 = loadChars(digits)
if (negative) p3 = "-$p3"
Token("$p1.${p2}e$p3", start, Token.Type.REAL)
} else {
// no exponent
Token("$p1.$p2", start, Token.Type.REAL)
}
} else {
// not decimal
// something like 10.times, method call on integer number
pos.back()
Token(p1, start, Token.Type.INT)
}
} else {
// could be integer, also hex:
if (currentChar == 'x' && p1 == "0") {
pos.advance()
val hex = loadChars { it in hexDigits }
Token(hex, start, Token.Type.HEX).also {
if (currentChar.isLetter())
raise("invalid hex literal")
}
} else {
Token(p1, start, Token.Type.INT)
}
}
private val currentChar: Char get() = pos.currentChar
private fun fixMultilineStringLiteral(source: String): String {
val sizes = mutableListOf<Int>()
val lines = source.lines().toMutableList()
if (lines.size == 0) return ""
if (lines[0].isBlank()) lines.removeFirst()
if (lines.isEmpty()) return ""
if (lines.last().isBlank()) lines.removeLast()
val normalized = lines.map { l ->
if (l.isBlank()) {
sizes.add(-1)
""
} else {
val margin = leftMargin(l)
sizes += margin
" ".repeat(margin) + l.trim()
}
}
val commonMargin = sizes.filter { it >= 0 }.min()
val fixed = if (commonMargin < 1) lines else normalized.map {
if (it.isBlank()) "" else it.drop(commonMargin)
}
return fixed.joinToString("\n")
}
private fun loadStringToken(): Token {
val start = currentPos
val sb = StringBuilder()
var newlineDetected = false
while (currentChar != '"') {
if (pos.end) throw ScriptError(start, "unterminated string started there")
when (currentChar) {
'\\' -> {
pos.advance() ?: raise("unterminated string")
when (currentChar) {
'n' -> {
sb.append('\n'); pos.advance()
}
'r' -> {
sb.append('\r'); pos.advance()
}
't' -> {
sb.append('\t'); pos.advance()
}
'"' -> {
sb.append('"'); pos.advance()
}
'\\' -> {
sb.append('\\'); pos.advance()
}
'u' -> {
sb.append(loadUnicodeEscape(start))
}
else -> {
sb.append('\\').append(currentChar)
pos.advance()
}
}
}
'\n', '\r' -> {
newlineDetected = true
sb.append(currentChar)
pos.advance()
}
else -> {
sb.append(currentChar)
pos.advance()
}
}
}
pos.advance()
val result = sb.toString().let { if (newlineDetected) fixMultilineStringLiteral(it) else it }
return Token(result, start, Token.Type.STRING)
}
private sealed interface StringChunk {
data class Literal(val text: String, val pos: Pos) : StringChunk
data class Expr(val tokens: List<Token>, val pos: Pos) : StringChunk
}
private fun loadStringTokens(startQuotePos: Pos): Token {
if (!interpolationEnabled) return loadStringToken()
val tokenPos = currentPos
val chunks = mutableListOf<StringChunk>()
val literal = StringBuilder()
var newlineDetected = false
var hasInterpolation = false
fun flushLiteralChunk() {
if (literal.isNotEmpty()) {
chunks += StringChunk.Literal(literal.toString(), tokenPos)
literal.clear()
}
}
while (currentChar != '"') {
if (pos.end) throw ScriptError(startQuotePos, "unterminated string started there")
when (currentChar) {
'\\' -> {
pos.advance() ?: raise("unterminated string")
when (currentChar) {
'n' -> {
literal.append('\n'); pos.advance()
}
'r' -> {
literal.append('\r'); pos.advance()
}
't' -> {
literal.append('\t'); pos.advance()
}
'"' -> {
literal.append('"'); pos.advance()
}
'\\' -> {
literal.append('\\'); pos.advance()
}
'u' -> {
literal.append(loadUnicodeEscape(tokenPos))
}
'$' -> {
// Backslash-escaped dollar is always literal.
literal.append('$')
pos.advance()
}
else -> {
literal.append('\\').append(currentChar)
pos.advance()
}
}
}
'$' -> {
pos.advance()
when {
currentChar == '$' -> {
// $$ -> literal '$'
literal.append('$')
pos.advance()
}
currentChar == '{' -> {
hasInterpolation = true
flushLiteralChunk()
val exprStart = pos.toPos()
pos.advance() // consume '{'
val exprText = readInterpolationExprText(startQuotePos)
val exprTokens = parseEmbeddedExpressionTokens(exprText, exprStart)
chunks += StringChunk.Expr(exprTokens, exprStart)
}
idFirstChars(currentChar) -> {
hasInterpolation = true
flushLiteralChunk()
val idPos = pos.toPos()
val id = loadChars(idNextChars)
val idToken = Token(id, idPos, Token.Type.ID)
chunks += StringChunk.Expr(listOf(idToken), idPos)
}
else -> {
// Bare '$' before non-interpolation text stays literal.
literal.append('$')
}
}
}
'\n', '\r' -> {
newlineDetected = true
literal.append(currentChar)
pos.advance()
}
else -> {
literal.append(currentChar)
pos.advance()
}
}
}
pos.advance() // closing quote
if (!hasInterpolation) {
val result = literal.toString().let { if (newlineDetected) fixMultilineStringLiteral(it) else it }
return Token(result, tokenPos, Token.Type.STRING)
}
flushLiteralChunk()
if (chunks.isEmpty()) {
return Token("", tokenPos, Token.Type.STRING)
}
val expanded = mutableListOf<Token>()
expanded += Token("(", tokenPos, Token.Type.LPAREN)
var emittedPieces = 0
for (chunk in chunks) {
val pieceTokens = when (chunk) {
is StringChunk.Literal -> {
if (chunk.text.isEmpty()) emptyList()
else listOf(Token(chunk.text, chunk.pos, Token.Type.STRING))
}
is StringChunk.Expr -> {
if (chunk.tokens.isEmpty()) throw ScriptError(chunk.pos, "empty interpolation expression")
if (chunk.tokens.size == 1) {
chunk.tokens
} else {
listOf(Token("(", chunk.pos, Token.Type.LPAREN)) +
chunk.tokens +
listOf(Token(")", chunk.pos, Token.Type.RPAREN))
}
}
}
if (pieceTokens.isEmpty()) continue
if (emittedPieces > 0) expanded += Token("+", tokenPos, Token.Type.PLUS)
expanded += pieceTokens
emittedPieces++
}
if (emittedPieces == 0) {
expanded += Token("", tokenPos, Token.Type.STRING)
}
expanded += Token(")", tokenPos, Token.Type.RPAREN)
val first = expanded.first()
for (i in 1 until expanded.size) bufferedTokens.addLast(expanded[i])
return first
}
private fun parseEmbeddedExpressionTokens(text: String, exprPos: Pos): List<Token> {
val tokens = parseLyng(Source(exprPos.source.fileName, text), interpolationEnabled)
if (tokens.isEmpty()) return emptyList()
val withoutEof = if (tokens.last().type == Token.Type.EOF) tokens.dropLast(1) else tokens
return withoutEof
}
private fun readInterpolationExprText(start: Pos): String {
val out = StringBuilder()
var depth = 1
while (!pos.end) {
val ch = currentChar
if (ch == '"') {
appendQuoted(out, '"')
continue
}
if (ch == '\'') {
appendQuoted(out, '\'')
continue
}
if (ch == '/' && peekChar() == '/') {
out.append('/').append('/')
pos.advance()
pos.advance()
while (!pos.end && currentChar != '\n') {
out.append(currentChar)
pos.advance()
}
continue
}
if (ch == '/' && peekChar() == '*') {
out.append('/').append('*')
pos.advance()
pos.advance()
var closed = false
while (!pos.end) {
val c = currentChar
if (c == '*' && peekChar() == '/') {
out.append('*').append('/')
pos.advance()
pos.advance()
closed = true
break
}
out.append(c)
pos.advance()
}
if (!closed) throw ScriptError(start, "unterminated block comment in interpolation")
continue
}
if (ch == '{') {
depth++
out.append(ch)
pos.advance()
continue
}
if (ch == '}') {
depth--
if (depth == 0) {
pos.advance() // consume closing '}'
return out.toString()
}
out.append(ch)
pos.advance()
continue
}
out.append(ch)
pos.advance()
}
throw ScriptError(start, "unterminated interpolation expression")
}
private fun appendQuoted(out: StringBuilder, quote: Char) {
out.append(quote)
pos.advance()
while (!pos.end) {
val c = currentChar
out.append(c)
pos.advance()
if (c == '\\') {
if (!pos.end) {
out.append(currentChar)
pos.advance()
}
continue
}
if (c == quote) return
}
}
private fun peekChar(): Char {
if (pos.end) return 0.toChar()
val mark = pos.toPos()
pos.advance()
val result = currentChar
pos.resetTo(mark)
return result
}
private fun loadUnicodeEscape(start: Pos): Char {
// Called when currentChar points to 'u' right after a backslash.
if (currentChar != 'u') throw ScriptError(currentPos, "expected unicode escape marker: u")
pos.advance() ?: throw ScriptError(start, "unterminated unicode escape")
var code = 0
repeat(4) {
val ch = currentChar
if (ch !in hexDigits) {
throw ScriptError(currentPos, "invalid unicode escape sequence, expected 4 hex digits")
}
code = (code shl 4) + ch.digitToInt(16)
pos.advance()
}
return code.toChar()
}
/**
* Load characters from the set until it reaches EOF or invalid character found.
* stop at EOF on character filtered by [isValidChar].
*
* Note this function loads only on one string. Multiline texts are not supported by
* this method.
*
* @return the string of valid characters, could be empty
*/
private fun loadChars(isValidChar: (Char) -> Boolean): String {
val startLine = pos.line
val result = StringBuilder()
while (!pos.end && pos.line == startLine) {
val ch = pos.currentChar
if (isValidChar(ch)) {
result.append(ch)
pos.advance()
} else
break
}
return result.toString()
}
@Suppress("unused")
private fun loadUntil(endChars: Set<Char>): String {
return if (pos.end) ""
else {
val result = StringBuilder()
while (!pos.end) {
val ch = pos.currentChar
if (ch in endChars) break
result.append(ch)
pos.advance()
}
result.toString()
}
}
private fun loadToEndOfLine(): String {
val result = StringBuilder()
// Read characters up to but not including the line break
while (!pos.end && pos.currentChar != '\n') {
result.append(pos.currentChar)
pos.advance()
}
return result.toString()
}
private fun loadTo(str: String): String? {
val result = StringBuilder()
while (!pos.readFragment(str)) {
if (pos.end) return null
result.append(pos.currentChar); pos.advance()
}
return result.toString()
}
/**
* next non-whitespace char (newline are skipped too) or null if EOF
*/
private fun skipws(): Char? {
while (!pos.end) {
val ch = pos.currentChar
if (ch == '\n') break
if (ch.isWhitespace())
pos.advance()
else
return ch
}
return null
}
init {
// skip shebang
if (pos.readFragment("#!"))
loadToEndOfLine()
}
}