Compare commits

..
2 Commits
20 changed files with 364 additions and 23 deletions

No files matched your search

+2
View File
@@ -43,6 +43,8 @@ Primary sources used: `lynglib/src/commonMain/kotlin/net/sergeych/lyng/{Parser,T
## 3. Literals and Core Expressions
- Scalars: `null`, `true`, `false`, `void`.
- List literals allow multiline layout, comments between entries and trailing commas.
Newlines before the closing `]` are whitespace, including in nested and empty lists.
- List literal: `[a, b, c]`, spreads with `...`.
- Spread positions: beginning, middle, end are all valid: `[...a]`, `[0, ...a, 4]`, `[head, ...mid, tail]`.
- Spread source must be a `List` at runtime (non-list spread raises an error).
+29
View File
@@ -42,6 +42,21 @@ suspend fun bootstrapHttp() {
}
```
Redirects are followed transparently, up to 10 hops by default. Hosts can set a different
limit when installing the module:
```kotlin
createHttpModule(
PermitAllHttpAccessPolicy,
scope,
LyngHttpConfig(maxRedirects = 5),
)
```
Exceeding the limit raises an `HTTP_REDIRECT_LIMIT_EXCEEDED` error containing the configured
limit, initial URL, and last redirect target. HTTPS-to-HTTP redirects remain disabled, and the
`Authorization` header is removed when a redirect changes authority.
---
## Using from Lyng scripts
@@ -179,3 +194,17 @@ val allowLocalOnly = object : HttpAccessPolicy {
- **Windows native:** supported via the Ktor WinHttp client backend
- **Apple native:** supported via the Ktor Darwin client backend
- **Other targets:** may report unsupported; use `Http.isSupported()` before relying on it
## Request failure diagnostics
Transport exceptions report the request method and target, a category, the
underlying exception/cause chain, and recovery advice. Query strings and userinfo
are redacted from the diagnostic target. Categories are `HTTP_TLS_FAILED`,
`HTTP_TIMEOUT`, `HTTP_DNS_FAILED`, `HTTP_CONNECTION_FAILED` and
`HTTP_REQUEST_FAILED`. Permission denials are `HTTP_ACCESS_DENIED`; redirect
limit errors retain `HTTP_REDIRECT_LIMIT_EXCEEDED`.
`HTTP_TLS_FAILED` means the handshake failed before an HTTP response. Do not
retry the same URL unchanged; use another source. A remote TLS fatal alert is
not a Lyng syntax error. Non-2xx responses remain ordinary response objects, and
request cancellation propagates rather than becoming a generic HTTP error.
+1 -1
View File
@@ -29,7 +29,7 @@ plugins {
}
group = "net.sergeych"
version = "0.0.1-SNAPSHOT"
version = providers.gradleProperty("lyngio.version").getOrElse("0.0.1-SNAPSHOT")
private fun Project.sqliteLinuxLinkerOpts(vararg defaultDirs: String): List<String> {
val overrideDir = providers.gradleProperty("sqlite3.lib.dir").orNull
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.cio.CIO
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(CIO)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(CIO, config)
@@ -17,6 +17,8 @@
package net.sergeych.lyng.io.http
import kotlinx.coroutines.CancellationException
import net.sergeych.lyng.ModuleScope
import net.sergeych.lyng.Scope
import net.sergeych.lyng.ScopeFacade
@@ -41,6 +43,7 @@ import net.sergeych.lyng.requireNoArgs
import net.sergeych.lyng.requireScope
import net.sergeych.lyngio.http.LyngHttpRequest
import net.sergeych.lyngio.http.LyngHttpResponse
import net.sergeych.lyngio.http.LyngHttpConfig
import net.sergeych.lyngio.http.getSystemHttpEngine
import net.sergeych.lyngio.http.security.HttpAccessDeniedException
import net.sergeych.lyngio.http.security.HttpAccessOp
@@ -51,21 +54,36 @@ import net.sergeych.lyngio.stdlib_included.http_typesLyng
private const val HTTP_MODULE_NAME = "lyng.io.http"
internal const val HTTP_TYPES_MODULE_NAME = "lyng.io.http.types"
fun createHttpModule(policy: HttpAccessPolicy, scope: Scope): Boolean =
createHttpModule(policy, scope.importManager)
fun createHttpModule(
policy: HttpAccessPolicy,
scope: Scope,
config: LyngHttpConfig = LyngHttpConfig(),
): Boolean = createHttpModule(policy, scope.importManager, config)
fun createHttp(policy: HttpAccessPolicy, scope: Scope): Boolean = createHttpModule(policy, scope)
fun createHttp(
policy: HttpAccessPolicy,
scope: Scope,
config: LyngHttpConfig = LyngHttpConfig(),
): Boolean = createHttpModule(policy, scope, config)
fun createHttpModule(policy: HttpAccessPolicy, manager: ImportManager): Boolean {
fun createHttpModule(
policy: HttpAccessPolicy,
manager: ImportManager,
config: LyngHttpConfig = LyngHttpConfig(),
): Boolean {
createHttpTypesModule(manager)
if (manager.packageNames.contains(HTTP_MODULE_NAME)) return false
manager.addPackage(HTTP_MODULE_NAME) { module ->
buildHttpModule(module, policy)
buildHttpModule(module, policy, config)
}
return true
}
fun createHttp(policy: HttpAccessPolicy, manager: ImportManager): Boolean = createHttpModule(policy, manager)
fun createHttp(
policy: HttpAccessPolicy,
manager: ImportManager,
config: LyngHttpConfig = LyngHttpConfig(),
): Boolean = createHttpModule(policy, manager, config)
internal fun createHttpTypesModule(manager: ImportManager): Boolean {
if (manager.packageNames.contains(HTTP_TYPES_MODULE_NAME)) return false
@@ -80,9 +98,9 @@ private suspend fun buildHttpTypesModule(module: ModuleScope) {
module.addConst("HttpHeaders", ObjHttpHeaders.type)
}
private suspend fun buildHttpModule(module: ModuleScope, policy: HttpAccessPolicy) {
private suspend fun buildHttpModule(module: ModuleScope, policy: HttpAccessPolicy, config: LyngHttpConfig) {
module.eval(Source(HTTP_MODULE_NAME, httpLyng))
val engine = getSystemHttpEngine()
val engine = getSystemHttpEngine(config)
val headersType = ObjHttpHeaders.type
val requestType = ObjHttpRequest.type
@@ -148,8 +166,10 @@ private suspend fun buildHttpModule(module: ModuleScope, policy: HttpAccessPolic
private suspend inline fun ScopeFacade.httpGuard(crossinline block: suspend () -> Obj): Obj {
return try {
block()
} catch (e: CancellationException) {
throw e
} catch (e: HttpAccessDeniedException) {
raiseIllegalOperation(e.reasonDetail ?: "http access denied")
raiseIllegalOperation("HTTP_ACCESS_DENIED: " + (e.reasonDetail ?: "http access denied"))
} catch (e: Exception) {
raiseIllegalOperation(e.message ?: "http error")
}
@@ -0,0 +1,100 @@
@file:OptIn(io.ktor.utils.io.InternalAPI::class)
package net.sergeych.lyngio.http
import io.ktor.client.HttpClient
import io.ktor.client.call.HttpClientCall
import io.ktor.client.plugins.HttpResponseRedirectEvent
import io.ktor.client.plugins.api.ClientPlugin
import io.ktor.client.plugins.api.Send
import io.ktor.client.plugins.api.createClientPlugin
import io.ktor.client.request.HttpRequestBuilder
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpMethod
import io.ktor.http.HttpStatusCode
import io.ktor.http.authority
import io.ktor.http.isSecure
import io.ktor.http.takeFrom
private val redirectMethods = setOf(HttpMethod.Get, HttpMethod.Head)
internal class HttpRedirectLimitExceededException(
val limit: Int,
val initialUrl: String,
val lastUrl: String,
) : IllegalStateException(
"HTTP_REDIRECT_LIMIT_EXCEEDED: limit=$limit, initialUrl=$initialUrl, lastUrl=$lastUrl"
)
internal class CappedHttpRedirectConfig {
var maxRedirects: Int = LyngHttpConfig.DEFAULT_MAX_REDIRECTS
var checkHttpMethod: Boolean = true
var allowHttpsDowngrade: Boolean = false
}
internal val CappedHttpRedirect: ClientPlugin<CappedHttpRedirectConfig> = createClientPlugin(
"CappedHttpRedirect",
::CappedHttpRedirectConfig,
) {
val maxRedirects = pluginConfig.maxRedirects.also {
require(it >= 0) { "maxRedirects must not be negative" }
}
val checkHttpMethod = pluginConfig.checkHttpMethod
val allowHttpsDowngrade = pluginConfig.allowHttpsDowngrade
suspend fun Send.Sender.followRedirects(
context: HttpRequestBuilder,
origin: HttpClientCall,
client: HttpClient,
): HttpClientCall {
var call = origin
var requestBuilder = context
val initialUrl = origin.request.url.toString()
var redirectCount = 0
while (call.response.status.isRedirectStatus()) {
val location = call.response.headers[HttpHeaders.Location] ?: return call
val previousProtocol = call.request.url.protocol
val previousAuthority = call.request.url.authority
requestBuilder = HttpRequestBuilder().apply {
takeFrom(requestBuilder)
url.parameters.clear()
url.takeFrom(location)
if (!allowHttpsDowngrade && previousProtocol.isSecure() && !url.protocol.isSecure()) {
return call
}
if (previousAuthority != url.authority) {
headers.remove(HttpHeaders.Authorization)
}
}
if (redirectCount >= maxRedirects) {
throw HttpRedirectLimitExceededException(
maxRedirects,
initialUrl,
requestBuilder.url.buildString(),
)
}
redirectCount += 1
client.monitor.raise(HttpResponseRedirectEvent, call.response)
call = proceed(requestBuilder)
}
return call
}
on(Send) { request ->
val origin = proceed(request)
if (checkHttpMethod && origin.request.method !in redirectMethods) {
return@on origin
}
followRedirects(request, origin, client)
}
}
private fun HttpStatusCode.isRedirectStatus(): Boolean = when (value) {
HttpStatusCode.MovedPermanently.value,
HttpStatusCode.Found.value,
HttpStatusCode.TemporaryRedirect.value,
HttpStatusCode.PermanentRedirect.value,
HttpStatusCode.SeeOther.value -> true
else -> false
}
@@ -0,0 +1,34 @@
package net.sergeych.lyngio.http
/** Transport diagnostics, preserved by the Lyng bridge instead of a generic "http error". */
class LyngHttpRequestException(request: LyngHttpRequest, cause: Throwable) :
IllegalStateException(describeHttpFailure(request, cause), cause)
internal fun describeHttpFailure(request: LyngHttpRequest, cause: Throwable): String {
val causes = mutableListOf<Throwable>()
var current: Throwable? = cause
repeat(8) {
val item = current ?: return@repeat
if (causes.any { it === item }) return@repeat
causes += item
current = item.cause
}
val details = causes.joinToString(" <- ") { "${it::class.simpleName ?: "Exception"}: ${it.message ?: "no detail"}" }
val text = details.lowercase()
val (code, advice) = when {
"http_redirect_limit_exceeded" in text -> "HTTP_REDIRECT_LIMIT_EXCEEDED" to
"Do not retry this URL unchanged; inspect the redirect chain or use another source."
"tls" in text || "ssl" in text || "handshake" in text || "certificate" in text -> "HTTP_TLS_FAILED" to
"The HTTPS handshake failed before an HTTP response. Do not retry this URL unchanged; changing its path/query will not repair TLS. Use another HTTPS host or source."
"timeout" in text || "timed out" in text -> "HTTP_TIMEOUT" to
"The request exceeded a time limit. At most a bounded retry may help; avoid repeating it indefinitely."
"unknownhost" in text || "unresolvedaddress" in text || "name or service not known" in text -> "HTTP_DNS_FAILED" to
"The hostname could not be resolved. Verify the hostname or use another source."
"connectexception" in text || "connection refused" in text || "connection reset" in text -> "HTTP_CONNECTION_FAILED" to
"The connection failed before a usable response. Verify the endpoint; avoid repeated unchanged requests."
else -> "HTTP_REQUEST_FAILED" to "Use the cause details to choose another source or a different approach."
}
// Query strings and userinfo can contain credentials; the script already knows the full URL.
val target = request.url.substringBefore('?').substringBefore('#').replace(Regex("//[^/@]+@"), "//[redacted]@")
return "$code: ${request.method.uppercase()} $target\nCause: $details\n$advice"
}
@@ -5,6 +5,8 @@ import io.ktor.client.call.body
import io.ktor.client.engine.HttpClientEngineConfig
import io.ktor.client.engine.HttpClientEngineFactory
import io.ktor.client.plugins.timeout
import io.ktor.client.plugins.HttpTimeout
import kotlinx.coroutines.CancellationException
import io.ktor.client.request.HttpRequestBuilder
import io.ktor.client.request.request
import io.ktor.client.request.setBody
@@ -14,14 +16,21 @@ import io.ktor.http.takeFrom
internal fun createKtorHttpEngine(
engineFactory: HttpClientEngineFactory<HttpClientEngineConfig>,
): LyngHttpEngine = KtorLyngHttpEngine(engineFactory)
config: LyngHttpConfig,
): LyngHttpEngine = KtorLyngHttpEngine(engineFactory, config)
private class KtorLyngHttpEngine(
engineFactory: HttpClientEngineFactory<HttpClientEngineConfig>,
config: LyngHttpConfig,
) : LyngHttpEngine {
private val clientResult = runCatching {
HttpClient(engineFactory) {
expectSuccess = false
followRedirects = false
install(HttpTimeout)
install(CappedHttpRedirect) {
maxRedirects = config.maxRedirects
}
}
}
@@ -33,15 +42,22 @@ private class KtorLyngHttpEngine(
throw UnsupportedOperationException(it.message ?: "HTTP client is not supported")
}
val response = httpClient.request {
applyRequest(request)
}
try {
val response = httpClient.request { applyRequest(request) }
return LyngHttpResponse(
status = response.status.value,
statusText = response.status.description,
headers = response.headers.entries().associate { it.key to it.value.toList() },
bodyBytes = response.body<ByteArray>(),
)
} catch (e: CancellationException) {
throw e
} catch (e: HttpRedirectLimitExceededException) {
// Keep the existing typed redirect-limit contract for host callers.
throw e
} catch (e: Exception) {
throw LyngHttpRequestException(request, e)
}
}
private fun HttpRequestBuilder.applyRequest(request: LyngHttpRequest) {
@@ -33,6 +33,18 @@ data class LyngHttpResponse(
val bodyBytes: ByteArray,
)
data class LyngHttpConfig(
val maxRedirects: Int = DEFAULT_MAX_REDIRECTS,
) {
init {
require(maxRedirects >= 0) { "maxRedirects must not be negative" }
}
companion object {
const val DEFAULT_MAX_REDIRECTS: Int = 10
}
}
interface LyngHttpEngine {
val isSupported: Boolean
suspend fun request(request: LyngHttpRequest): LyngHttpResponse
@@ -46,4 +58,4 @@ internal object UnsupportedHttpEngine : LyngHttpEngine {
}
}
expect fun getSystemHttpEngine(): LyngHttpEngine
expect fun getSystemHttpEngine(config: LyngHttpConfig = LyngHttpConfig()): LyngHttpEngine
@@ -0,0 +1,25 @@
package net.sergeych.lyngio.http
import kotlin.test.*
class HttpRequestFailureTest {
private val request = LyngHttpRequest("GET", "https://user:secret@example.com/page?key=private#fragment")
@Test fun savedFatalTlsAlertGetsSpecificCauseAndNoUnchangedRetryAdvice() {
val saved = IllegalStateException("Received alert during handshake. Level: FATAL, code: InternalError")
val message = LyngHttpRequestException(request, saved).message!!
assertTrue(message.startsWith("HTTP_TLS_FAILED: GET https://[redacted]@example.com/page"))
assertTrue(message.contains("FATAL, code: InternalError"))
assertTrue(message.contains("Do not retry this URL unchanged"))
assertTrue(message.contains("changing its path/query will not repair TLS"))
assertFalse(message.contains("secret")); assertFalse(message.contains("private"))
}
@Test fun nestedCausesAndTimeoutDnsAndConnectionFailuresRemainDistinct() {
val nested = IllegalStateException("request failed", IllegalArgumentException("TLS certificate rejected"))
assertTrue(describeHttpFailure(request, nested).contains("HTTP_TLS_FAILED"))
for ((detail, code) in listOf("request timed out" to "HTTP_TIMEOUT",
"UnknownHostException: example.com" to "HTTP_DNS_FAILED",
"Connection refused" to "HTTP_CONNECTION_FAILED")) {
assertTrue(describeHttpFailure(request, IllegalStateException(detail)).startsWith(code))
}
}
}
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.darwin.Darwin
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(Darwin)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(Darwin, config)
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.js.Js
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(Js)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(Js, config)
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.cio.CIO
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(CIO)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(CIO, config)
@@ -0,0 +1,73 @@
package net.sergeych.lyngio.http
import com.sun.net.httpserver.HttpExchange
import com.sun.net.httpserver.HttpServer
import io.ktor.client.engine.cio.CIO
import kotlinx.coroutines.runBlocking
import java.net.InetSocketAddress
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
class KtorHttpRedirectTest {
@Test
fun redirectsWithinLimitRemainTransparent() = runBlocking {
val server = newServer { exchange ->
when (exchange.requestURI.path) {
"/start" -> redirect(exchange, "/middle")
"/middle" -> redirect(exchange, "/result")
else -> writeResponse(exchange, 200, "redirected")
}
}
try {
val engine = createKtorHttpEngine(CIO, LyngHttpConfig(maxRedirects = 2))
val response = engine.request(LyngHttpRequest("GET", url(server, "/start")))
assertEquals(200, response.status)
assertEquals("redirected", response.bodyBytes.decodeToString())
} finally {
server.stop(0)
}
}
@Test
fun redirectLimitProducesDiagnosticError() = runBlocking {
val server = newServer { exchange ->
val next = exchange.requestURI.path.removePrefix("/").toIntOrNull()?.plus(1) ?: 1
redirect(exchange, "/$next")
}
try {
val engine = createKtorHttpEngine(CIO, LyngHttpConfig(maxRedirects = 2))
val error = assertFailsWith<HttpRedirectLimitExceededException> {
engine.request(LyngHttpRequest("GET", url(server, "/0")))
}
assertEquals(2, error.limit)
assertTrue(error.initialUrl.endsWith("/0"), error.initialUrl)
assertTrue(error.lastUrl.endsWith("/3"), error.lastUrl)
assertTrue(error.message.orEmpty().startsWith("HTTP_REDIRECT_LIMIT_EXCEEDED:"), error.message)
} finally {
server.stop(0)
}
}
private fun newServer(handler: (HttpExchange) -> Unit): HttpServer =
HttpServer.create(InetSocketAddress("127.0.0.1", 0), 0).apply {
createContext("/") { exchange -> handler(exchange) }
start()
}
private fun url(server: HttpServer, path: String) =
"http://127.0.0.1:${server.address.port}$path"
private fun redirect(exchange: HttpExchange, location: String) {
exchange.responseHeaders.add("Location", location)
exchange.sendResponseHeaders(302, -1)
exchange.close()
}
private fun writeResponse(exchange: HttpExchange, status: Int, body: String) {
val bytes = body.encodeToByteArray()
exchange.sendResponseHeaders(status, bytes.size.toLong())
exchange.responseBody.use { it.write(bytes) }
}
}
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.curl.Curl
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(Curl)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(Curl, config)
@@ -2,4 +2,4 @@ package net.sergeych.lyngio.http
import io.ktor.client.engine.winhttp.WinHttp
actual fun getSystemHttpEngine(): LyngHttpEngine = createKtorHttpEngine(WinHttp)
actual fun getSystemHttpEngine(config: LyngHttpConfig): LyngHttpEngine = createKtorHttpEngine(WinHttp, config)
+9 -1
View File
@@ -32,7 +32,15 @@ extern class HttpResponse {
fun bytes(): Buffer
}
/* HTTP/HTTPS client singleton. */
/*
HTTP/HTTPS client singleton.
Transport failures include a diagnostic code, request method/URL, underlying cause,
and recovery advice: HTTP_TLS_FAILED, HTTP_TIMEOUT, HTTP_DNS_FAILED,
HTTP_CONNECTION_FAILED, HTTP_REQUEST_FAILED. HTTP_ACCESS_DENIED means no grant.
HTTP_TLS_FAILED occurs before any response: do not retry the URL unchanged;
use another source. Query strings/userinfo are redacted from the diagnostic target.
Non-2xx HTTP statuses are returned normally. Cancellation propagates to the host.
*/
extern object Http {
/* True when HTTP client support is available on this runtime. */
fun isSupported(): Bool
+1 -1
View File
@@ -21,7 +21,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
group = "net.sergeych"
version = "1.5.10"
version = providers.gradleProperty("lyng.version").getOrElse("1.5.10")
// Removed legacy buildscript classpath declarations; plugins are applied via the plugins DSL below
@@ -4021,6 +4021,7 @@ class Compiler(
while (true) {
val t = cc.next()
when (t.type) {
Token.Type.NEWLINE, Token.Type.SINGLE_LINE_COMMENT, Token.Type.MULTILINE_COMMENT -> continue
Token.Type.COMMA -> {
// todo: check commas sequences like [,] [,,] before, after or instead of expressions
}
@@ -0,0 +1,21 @@
import kotlinx.coroutines.test.runTest
import net.sergeych.lyng.Script
import kotlin.test.*
class MultilineListLiteralTest {
@Test fun finalNewlineCommentsAndTrailingCommaAreWhitespace() = runTest {
val scope = Script.newScope()
for (source in listOf(
"[\n \"first\",\n \"second\"\n]",
"[\n \"first\",\n \"second\",\n]",
"[\n // before\n \"first\", /* between */\n \"second\" // after\n]",
)) assertEquals("[first,second]", scope.eval(source).toString(scope).value)
}
@Test fun nestedAndEmptyMultilineListsAndExpressionsRemainValid() = runTest {
val scope = Script.newScope()
assertEquals("[]", scope.eval("[\n]").toString(scope).value)
assertEquals("[[1,2],[3,4]]", scope.eval("[\n [1,2],\n [3,4]\n]").toString(scope).value)
assertEquals("[3,7]", scope.eval("[\n 1+2,\n 3+4\n]").toString(scope).value)
assertFails { scope.eval("[\n @\n]") }
}
}