SergeychWorks/lyng
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases 21 Wiki Activity

Add backtick string literals and formatter support

Browse Source
This commit is contained in:
Sergey Chernov 2026-04-13 23:08:18 +03:00
parent e925195495
commit fada848907
15 changed files with 273 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
docs/ai_language_reference.md
View File

@ -15,15 +15,16 @@ Primary sources used: `lynglib/src/commonMain/kotlin/net/sergeych/lyng/{Parser,T
## 2. Lexical Syntax
- Comments: `// line`, `/* block */`.
- Strings: `"..."` (supports escapes). Multiline string content is normalized by indentation logic.
- Supported escapes: `\n`, `\r`, `\t`, `\"`, `\\`, `\uXXXX` (4 hex digits).
- Strings: `"..."` or `` `...` `` (supports escapes). Multiline string content is normalized by indentation logic.
- Shared escapes: `\n`, `\r`, `\t`, `\\`, `\uXXXX` (4 hex digits).
- Delimiter escapes: `\"` inside `"..."`, ``\` `` inside `` `...` ``.
- Unicode escapes use exactly 4 hex digits (for example: `"\u0416"` -> `Ж`).
- Unknown `\x` escapes in strings are preserved literally as two characters (`\` and `x`).
- String interpolation is supported:
- identifier form: `"$name"`
- expression form: `"${expr}"`
- escaped dollar: `"\$"` and `"$$"` both produce literal `$`.
- `\\$x` means backslash + interpolated `x`.
- identifier form: `"$name"` or `` `$name` ``
- expression form: `"${expr}"` or `` `${expr}` ``
- escaped dollar: `"\$"`, `"$$"`, `` `\$` ``, and `` `$$` `` all produce literal `$`.
- `\\$x` means backslash + interpolated `x` in either delimiter form.
- Per-file opt-out is supported via leading comment directive:
- `// feature: interpolation: off`
- with this directive, `$...` stays literal text.

19
docs/tutorial.md
View File

@ -1654,15 +1654,27 @@ The type for the character objects is `Char`.
### String literal escapes
Lyng string literals can use either double quotes or backticks:
val a = "hello"
val b = `hello`
assert(a == b)
| escape | ASCII value |
|--------|-----------------------|
| \n | 0x10, newline |
| \r | 0x13, carriage return |
| \t | 0x07, tabulation |
| \\ | \ slash character |
| \" | " double quote |
| \uXXXX | unicode code point |
Delimiter-specific escapes:
| form | escape | value |
|--------|--------|------------------|
| `"..."` | \" | " double quote |
| `` `...` `` | \` | ` backtick |
Unicode escape form is exactly 4 hex digits, e.g. `"\u263A"` -> `☺`.
Other `\c` combinations, where c is any char except mentioned above, are left intact, e.g.:
@ -1695,10 +1707,15 @@ Example:
val name = "Lyng"
assertEquals("hello, Lyng!", "hello, $name!")
assertEquals("hello, Lyng!", `hello, $name!`)
assertEquals("sum=3", "sum=${1+2}")
assertEquals("sum=3", `sum=${1+2}`)
assertEquals("\$name", "\$name")
assertEquals("\$name", "$$name")
assertEquals("\$name", `\$name`)
assertEquals("\$name", `$$name`)
assertEquals("\\Lyng", "\\$name")
assertEquals("\\Lyng", `\\$name`)
>>> void
Interpolation and `printf`-style formatting can be combined when needed:

2
lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/docs/LyngDocumentationProvider.kt
View File

@ -310,7 +310,7 @@ class LyngDocumentationProvider : AbstractDocumentationProvider() {
// Try literal and call-based receiver inference around the dot
val i = TextCtx.prevNonWs(text, dotPos - 1)
val className: String? = when {
i >= 0 && text[i] == '"' -> "String"
i >= 0 && (text[i] == '"' || text[i] == '`') -> "String"
i >= 0 && text[i] == ']' -> "List"
i >= 0 && text[i] == '}' -> "Dict"
i >= 0 && text[i] == ')' -> {

3
lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/format/LyngPreFormatProcessor.kt
View File

@ -24,6 +24,7 @@ import com.intellij.psi.codeStyle.CodeStyleManager
import com.intellij.psi.impl.source.codeStyle.PreFormatProcessor
import net.sergeych.lyng.format.LyngFormatConfig
import net.sergeych.lyng.format.LyngFormatter
import net.sergeych.lyng.format.LyngStringDelimiterPolicy
import net.sergeych.lyng.idea.LyngLanguage
/**
@ -170,6 +171,7 @@ class LyngPreFormatProcessor : PreFormatProcessor {
continuationIndentSize = options.CONTINUATION_INDENT_SIZE.coerceAtLeast(options.INDENT_SIZE.coerceAtLeast(1)),
applySpacing = true,
applyWrapping = false,
stringDelimiterPolicy = LyngStringDelimiterPolicy.PreferFewerEscapes,
)
val r = if (runFullFileIndent) currentLocalRange() else workingRangeLocal.intersection(currentLocalRange()) ?: currentLocalRange()
val text = doc.getText(r)
@ -189,6 +191,7 @@ class LyngPreFormatProcessor : PreFormatProcessor {
continuationIndentSize = options.CONTINUATION_INDENT_SIZE.coerceAtLeast(options.INDENT_SIZE.coerceAtLeast(1)),
applySpacing = settings.enableSpacing,
applyWrapping = true,
stringDelimiterPolicy = LyngStringDelimiterPolicy.PreferFewerEscapes,
)
val r = if (runFullFileIndent) currentLocalRange() else workingRangeLocal.intersection(currentLocalRange()) ?: currentLocalRange()
val text = doc.getText(r)

4
lyng-idea/src/main/kotlin/net/sergeych/lyng/idea/highlight/LyngLexer.kt
View File

@ -101,8 +101,8 @@ class LyngLexer : LexerBase() {
return
}
// String "..." or '...' with simple escape handling
if (ch == '"' || ch == '\'') {
// String "...", `...`, or '...' with simple escape handling
if (ch == '"' || ch == '\'' || ch == '`') {
val quote = ch
i++
while (i < endOffset) {

58
lyng-idea/src/test/kotlin/net/sergeych/lyng/idea/highlight/LyngLexerBacktickStringTest.kt Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright 2026 Sergey S. Chernov real.sergeych@gmail.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package net.sergeych.lyng.idea.highlight
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Test
class LyngLexerBacktickStringTest {
@Test
fun backtickStringGetsStringTokenAndColor() {
val lexer = LyngLexer()
val source = """val json = `{"name":"lyng","doc":"use \`quotes\`"}`"""
lexer.start(source, 0, source.length, 0)
val tokens = mutableListOf<Pair<String, String>>()
while (lexer.tokenType != null) {
val tokenText = source.substring(lexer.tokenStart, lexer.tokenEnd)
tokens += lexer.tokenType.toString() to tokenText
lexer.advance()
}
assertEquals(
listOf(
"KEYWORD" to "val",
"WHITESPACE" to " ",
"IDENTIFIER" to "json",
"WHITESPACE" to " ",
"PUNCT" to "=",
"WHITESPACE" to " ",
"STRING" to "`{\"name\":\"lyng\",\"doc\":\"use \\`quotes\\`\"}`"
),
tokens
)
val highlighter = LyngSyntaxHighlighter()
assertArrayEquals(
arrayOf(LyngHighlighterColors.STRING),
highlighter.getTokenHighlights(LyngTokenTypes.STRING)
)
}
}

1
lyng/src/commonMain/kotlin/Common.kt
View File

@ -420,6 +420,7 @@ private class Fmt : CoreCliktCommand(name = "fmt") {
val cfg = net.sergeych.lyng.format.LyngFormatConfig(
applySpacing = enableSpacing,
applyWrapping = enableWrapping,
stringDelimiterPolicy = net.sergeych.lyng.format.LyngStringDelimiterPolicy.PreferFewerEscapes,
)
var anyChanged = false

24
lynglib/src/commonMain/kotlin/net/sergeych/lyng/Parser.kt
View File

@ -361,7 +361,7 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
Token(":", from, Token.Type.COLON)
}
'"' -> loadStringTokens(from)
'"', '`' -> loadStringTokens(from, ch)
in digitsSet -> {
pos.back()
@ -550,11 +550,11 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
return fixed.joinToString("\n")
}
private fun loadStringToken(): Token {
private fun loadStringToken(delimiter: Char): Token {
val start = currentPos
val sb = StringBuilder()
var newlineDetected = false
while (currentChar != '"') {
while (currentChar != delimiter) {
if (pos.end) throw ScriptError(start, "unterminated string started there")
when (currentChar) {
'\\' -> {
@ -572,8 +572,8 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
sb.append('\t'); pos.advance()
}
'"' -> {
sb.append('"'); pos.advance()
delimiter -> {
sb.append(delimiter); pos.advance()
}
'\\' -> {
@ -615,8 +615,8 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
data class Expr(val tokens: List<Token>, val pos: Pos) : StringChunk
}
private fun loadStringTokens(startQuotePos: Pos): Token {
if (!interpolationEnabled) return loadStringToken()
private fun loadStringTokens(startQuotePos: Pos, delimiter: Char): Token {
if (!interpolationEnabled) return loadStringToken(delimiter)
val tokenPos = currentPos
val chunks = mutableListOf<StringChunk>()
@ -631,7 +631,7 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
}
}
while (currentChar != '"') {
while (currentChar != delimiter) {
if (pos.end) throw ScriptError(startQuotePos, "unterminated string started there")
when (currentChar) {
'\\' -> {
@ -649,8 +649,8 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
literal.append('\t'); pos.advance()
}
'"' -> {
literal.append('"'); pos.advance()
delimiter -> {
literal.append(delimiter); pos.advance()
}
'\\' -> {
@ -788,8 +788,8 @@ private class Parser(fromPos: Pos, private val interpolationEnabled: Boolean = t
var depth = 1
while (!pos.end) {
val ch = currentChar
if (ch == '"') {
appendQuoted(out, '"')
if (ch == '"' || ch == '`') {
appendQuoted(out, ch)
continue
}
if (ch == '\'') {

6
lynglib/src/commonMain/kotlin/net/sergeych/lyng/format/LyngFormatConfig.kt
View File

@ -16,6 +16,11 @@
*/
package net.sergeych.lyng.format
enum class LyngStringDelimiterPolicy {
Preserve,
PreferFewerEscapes,
}
/**
* Formatting configuration for Lyng source code.
* Defaults are Kotlin-like.
@ -28,6 +33,7 @@ data class LyngFormatConfig(
val applySpacing: Boolean = false,
val applyWrapping: Boolean = false,
val trailingComma: Boolean = false,
val stringDelimiterPolicy: LyngStringDelimiterPolicy = LyngStringDelimiterPolicy.Preserve,
) {
init {
require(indentSize > 0) { "indentSize must be > 0" }

105
lynglib/src/commonMain/kotlin/net/sergeych/lyng/format/LyngFormatter.kt
View File

@ -274,7 +274,9 @@ object LyngFormatter {
fun format(text: String, config: LyngFormatConfig = LyngFormatConfig()): String {
// Phase 1: indentation
val indented = reindent(text, config)
if (!config.applySpacing && !config.applyWrapping) return indented
if (!config.applySpacing && !config.applyWrapping &&
config.stringDelimiterPolicy == LyngStringDelimiterPolicy.Preserve
) return indented
// Phase 2: minimal, safe spacing (PSI-free).
val lines = indented.split('\n')
@ -286,14 +288,27 @@ object LyngFormatter {
val (parts, nextInBlockComment) = splitIntoParts(rawLine, inBlockComment)
val sb = StringBuilder()
for (part in parts) {
if (part.type == PartType.Code) {
sb.append(applyMinimalSpacingRules(part.text))
} else {
sb.append(part.text)
val normalizedPart = when (part.type) {
PartType.Code -> if (config.applySpacing) applyMinimalSpacingRules(part.text) else part.text
PartType.StringLiteral -> applyStringLiteralPolicy(part.text, config.stringDelimiterPolicy)
else -> part.text
}
sb.append(normalizedPart)
}
line = sb.toString()
inBlockComment = nextInBlockComment
} else if (config.stringDelimiterPolicy != LyngStringDelimiterPolicy.Preserve) {
val (parts, nextInBlockComment) = splitIntoParts(rawLine, inBlockComment)
line = buildString(rawLine.length) {
for (part in parts) {
append(
if (part.type == PartType.StringLiteral) {
applyStringLiteralPolicy(part.text, config.stringDelimiterPolicy)
} else part.text
)
}
}
inBlockComment = nextInBlockComment
}
out.append(line.trimEnd())
if (i < lines.lastIndex) out.append('\n')
@ -463,6 +478,84 @@ object LyngFormatter {
private enum class PartType { Code, StringLiteral, BlockComment, LineComment }
private data class Part(val text: String, val type: PartType)
private fun applyStringLiteralPolicy(text: String, policy: LyngStringDelimiterPolicy): String {
if (policy == LyngStringDelimiterPolicy.Preserve) return text
if (text.length < 2) return text
val delimiter = text.first()
if (delimiter != '"' && delimiter != '`') return text
if (text.last() != delimiter) return text
val other = if (delimiter == '"') '`' else '"'
val rewritten = rewriteStringLiteralDelimiter(text, other) ?: return text
return when (policy) {
LyngStringDelimiterPolicy.Preserve -> text
LyngStringDelimiterPolicy.PreferFewerEscapes -> {
val currentCost = delimiterEscapeCost(text, delimiter)
val rewrittenCost = delimiterEscapeCost(rewritten, other)
if (rewrittenCost < currentCost) rewritten else text
}
}
}
private fun delimiterEscapeCost(text: String, delimiter: Char): Int {
var cost = 0
var i = 1
while (i < text.length - 1) {
val ch = text[i]
if (ch == '\\' && i + 1 < text.length - 1) {
val next = text[i + 1]
if (next == delimiter) cost++
i += 2
continue
}
if (ch == delimiter) cost++
i++
}
return cost
}
private fun rewriteStringLiteralDelimiter(text: String, targetDelimiter: Char): String? {
if (text.length < 2) return null
val sourceDelimiter = text.first()
if ((sourceDelimiter != '"' && sourceDelimiter != '`') || text.last() != sourceDelimiter) return null
if (sourceDelimiter == targetDelimiter) return text
val body = StringBuilder(text.length + 8)
var i = 1
val end = text.length - 1
while (i < end) {
val ch = text[i]
if (ch == '\\' && i + 1 < end) {
val next = text[i + 1]
when {
next == sourceDelimiter -> {
if (sourceDelimiter == targetDelimiter) body.append('\\').append(targetDelimiter)
else body.append(next)
i += 2
}
next == targetDelimiter -> {
body.append('\\').append('\\').append('\\').append(targetDelimiter)
i += 2
}
else -> {
body.append(ch).append(next)
i += 2
}
}
continue
}
if (ch == targetDelimiter) {
body.append('\\').append(targetDelimiter)
} else {
body.append(ch)
}
i++
}
return buildString(body.length + 2) {
append(targetDelimiter)
append(body)
append(targetDelimiter)
}
}
/**
* Split a line into parts: code, string literals, and comments.
* Tracks [inBlockComment] state across lines.
@ -514,7 +607,7 @@ private fun splitIntoParts(
inBlockComment = true
last = i
i += 2
} else if (text[i] == '"' || text[i] == '\'') {
} else if (text[i] == '"' || text[i] == '\'' || text[i] == '`') {
if (i > last) result.add(Part(text.substring(last, i), PartType.Code))
inString = true
quoteChar = text[i]

5
lynglib/src/commonMain/kotlin/net/sergeych/lyng/highlight/SimpleLyngHighlighter.kt
View File

@ -143,7 +143,10 @@ class SimpleLyngHighlighter : LyngHighlighter {
val k = kindOf(t.type, t.value) ?: continue
val start0 = src.offsetOf(t.pos)
val range = when (t.type) {
Type.STRING, Type.STRING2 -> adjustQuoteSpan(start0, '"')
Type.STRING, Type.STRING2 -> {
val quote = text.getOrNull(start0)?.takeIf { it == '"' || it == '`' } ?: '"'
adjustQuoteSpan(start0, quote)
}
Type.CHAR -> adjustQuoteSpan(start0, '\'')
Type.HEX -> {
// Parser returns HEX token value without the leading "0x"; include it in highlight span

2
lynglib/src/commonMain/kotlin/net/sergeych/lyng/miniast/DocLookupUtils.kt
View File

@ -530,7 +530,7 @@ object DocLookupUtils {
var inString = false
while (i < text.length) {
val ch = text[i]
if (ch == '"' && (i == 0 || text[i - 1] != '\\')) {
if ((ch == '"' || ch == '`') && (i == 0 || text[i - 1] != '\\')) {
inString = !inString
}
if (!inString && ch == '/' && i + 1 < text.length) {

24
lynglib/src/commonTest/kotlin/ScriptTest.kt
View File

@ -4395,6 +4395,30 @@ class ScriptTest {
)
}
@Test
fun backtickStringsMatchRegularStringSemantics() = runTest {
val d = '$'
eval(
"""
val name = "Lyng"
val simple = `hello, $d{name}`
assertEquals("hello, Lyng", simple)
assertEquals("{\"name\":\"Lyng\"}", `{"name":"Lyng"}`)
assertEquals("use the `code` style", `use the \`code\` style`)
assertEquals("\\\"", `\"`)
assertEquals("\"", `"`)
assertEquals(
"first\n\"second\"\nthird",
`
first
"second"
third
`
)
""".trimIndent()
)
}
@Test
fun testInlineArrayLiteral() = runTest {
eval(

8
lynglib/src/commonTest/kotlin/UnicodeEscapeTest.kt
View File

@ -31,6 +31,13 @@ class UnicodeEscapeTest {
assertEquals("☺", token.value)
}
@Test
fun parserDecodesUnicodeEscapeInBacktickStringLiteral() {
val token = parseLyng("`\\u263A`".toSource()).first()
assertEquals(Token.Type.STRING, token.type)
assertEquals("☺", token.value)
}
@Test
fun parserDecodesUnicodeEscapeInCharLiteral() {
val token = parseLyng("'\\u263A'".toSource()).first()
@ -55,6 +62,7 @@ class UnicodeEscapeTest {
@Test
fun evalDecodesUnicodeEscapes() = runTest {
assertEquals(ObjString("☺"), eval("\"\\u263A\""))
assertEquals(ObjString("☺"), eval("`\\u263A`"))
assertEquals(ObjChar('☺'), eval("'\\u263A'"))
}
}

31
lynglib/src/commonTest/kotlin/net/sergeych/lyng/format/LyngFormatterTest.kt
View File

@ -18,9 +18,36 @@ package net.sergeych.lyng.format
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class LyngFormatterTest {
@Test
fun preferBackticksForQuoteHeavyStrings() {
val src = "val json = \"{\\\"name\\\":\\\"lyng\\\",\\\"kind\\\":\\\"lang\\\"}\""
val cfg = LyngFormatConfig(
applySpacing = true,
stringDelimiterPolicy = LyngStringDelimiterPolicy.PreferFewerEscapes
)
val out = LyngFormatter.format(src, cfg)
assertEquals("""val json = `{"name":"lyng","kind":"lang"}`""", out)
assertEquals(out, LyngFormatter.format(out, cfg))
}
@Test
fun preserveStringsWhenAlternativeWouldNotHelp() {
val src = "val sample = \"use `ticks` and keep \\` literal\""
val cfg = LyngFormatConfig(
stringDelimiterPolicy = LyngStringDelimiterPolicy.PreferFewerEscapes
)
val out = LyngFormatter.format(src, cfg)
assertEquals(src, out)
}
@Test
fun labelFormatting() {
val src = "return @label; break @outer; continue @inner"
@ -79,9 +106,9 @@ class LyngFormatterTest {
val formatted = LyngFormatter.format(src, LyngFormatConfig(applyWrapping = true, maxLineLength = 40, continuationIndentSize = 4))
// Ensure the string literal remains intact
kotlin.test.assertTrue(formatted.contains(arg2), "String literal must be preserved")
assertTrue(formatted.contains(arg2), "String literal must be preserved")
// Ensure end-of-line comment remains
kotlin.test.assertTrue(formatted.contains("// end comment"), "EOL comment must be preserved")
assertTrue(formatted.contains("// end comment"), "EOL comment must be preserved")
// Idempotency
val formatted2 = LyngFormatter.format(formatted, LyngFormatConfig(applyWrapping = true, maxLineLength = 40, continuationIndentSize = 4))
assertEquals(formatted, formatted2)