Add offline HTML parsing module to LyngIO

.gitignore

@@ -32,3 +32,4 @@ test_output*.txt
 /bugs/
 contents.db
 /docs/idea_plugin.md.deploy_backup
+/.claude/settings.local.json

README.md

@@ -55,6 +55,7 @@ assertEquals(A.E.One, A.One)
 - [SQL Databases (lyng.io.db)](docs/lyng.io.db.md)
 - [HTTP Server (lyng.io.http.server)](docs/lyng.io.http.server.md)
 - [HTML Builder (lyng.io.html)](docs/lyng.io.html.md)
+- [HTML Parsing (lyng.io.html.parse)](docs/lyng.io.html.parse.md)
 - [Process Execution (lyng.io.process)](docs/lyng.io.process.md)
 - [Time and Calendar Types](docs/time.md)
 - [Json support](docs/json_and_kotlin_serialization.md)

docs/ai_stdlib_reference.md

@@ -98,6 +98,7 @@ Requires installing `lyngio` into the import manager from host code.
 - `import lyng.io.ws` (WebSocket client API; currently supported on JVM, capability-gated elsewhere)
 - `import lyng.io.net` (TCP/UDP transport API; currently supported on JVM, capability-gated elsewhere)
 - `import lyng.io.html` (pure Lyng HTML builder DSL: `html { body { h3 { +"text" } } }`)
+- `import lyng.io.html.parse` (offline HTML5 parsing: `Html.parse(source, baseUrl = "")`, document title/text, CSS selectors, element text/attributes/absolute URLs; see [HTML parsing](lyng.io.html.parse.md))
 - Shared network value-type packages are also available when installed by host code:
   - `import lyng.io.http.types` (`HttpHeaders`)
   - `import lyng.io.ws.types` (`WsMessage`)

docs/lyng.io.html.md

@@ -1,5 +1,7 @@
 # lyng.io.html
 
+For reading existing HTML, see [lyng.io.html.parse](lyng.io.html.parse.md).
+
 `lyng.io.html` provides a pure Lyng HTML builder DSL. It uses Lyng context
 receiver extensions, so text can be appended with `+"text"` inside tag blocks
 without global builder state.

docs/lyng.io.html.parse.md

@@ -0,0 +1,41 @@
+# lyng.io.html.parse
+
+Offline HTML5 parsing backed by the core [Ksoup](https://github.com/fleeksoft/ksoup)
+parser (no network extensions). Installed from `lyngio`:
+
+```kotlin
+createHtmlParseModule(scope.importManager)
+```
+
+```lyng
+import lyng.io.html.parse
+
+val doc = Html.parse("<title>News</title><main><p>Hello &amp; welcome</p><a href='/next'>Next</a></main>", "https://example.com/")
+println(doc.title)
+println(doc.text())
+for (raw in doc.select("a[href]")) {
+    val link = raw as HtmlElement
+    println(link.text() + " => " + link.absUrl("href"))
+}
+```
+
+`Html.parse(source, baseUrl = "")` accepts documents, fragments and malformed HTML.
+It returns `HtmlDocument`, with `title: String`, `text(): String` and
+`select(css): List<HtmlElement>`. Elements have `text()`, `attr(name)`,
+`absUrl(name)` and `select(css)`. Missing selections return an empty list;
+missing attributes or unresolvable absolute URLs return an empty string.
+Invalid CSS selectors raise an error. `<base href>` participates in URL resolution.
+Use the final fetched URL as the base when available.
+
+Text extraction decodes entities, normalizes whitespace, preserves block and
+`br` boundaries as newlines, and excludes `script`, `style`, `template`, `noscript`
+and `[hidden]` subtrees. Selection retains the original DOM. This is readable
+text extraction, not CSS rendering or article detection; navigation and cookie
+banners may remain. It does not preserve preformatted whitespace.
+
+Parsing never fetches resources or executes JavaScript. `absUrl` only resolves
+URLs; it does not filter schemes. Fetch through the host's authorized HTTP API,
+check status and content type, and choose bounded excerpts to return to an AI.
+This API parses HTML only; XML parsing is separate future work.
+
+Canonical declarations: `lyngio/stdlib/lyng/io/html_parse.lyng`.

lyngio/build.gradle.kts

@@ -29,7 +29,7 @@ plugins {
 }
 
 group = "net.sergeych"
-version = providers.gradleProperty("lyngio.version").getOrElse("0.0.1-SNAPSHOT")
+version = providers.gradleProperty("lyngio.version").getOrElse("0.0.3-SNAPSHOT")
 
 private fun Project.sqliteLinuxLinkerOpts(vararg defaultDirs: String): List<String> {
     val overrideDir = providers.gradleProperty("sqlite3.lib.dir").orNull
@@ -141,6 +141,7 @@ kotlin {
                 api(libs.kotlinx.coroutines.core)
                 api(libs.mordant.core)
                 api(libs.ktor.client.core)
+                implementation("com.fleeksoft.ksoup:ksoup:0.2.6")
                 implementation(libs.ktor.client.websockets)
             }
         }
@@ -355,3 +356,9 @@ publishing {
         }
     }
 }
+
+// Lyng's JVM dependencies include Java 21 bytecode; keep the Java 17 library target.
+val testToolchains = extensions.getByType<org.gradle.jvm.toolchain.JavaToolchainService>()
+tasks.withType<Test>().configureEach {
+    javaLauncher.set(testToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(21)) })
+}

lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModule.kt

@@ -0,0 +1,96 @@
+package net.sergeych.lyng.io.html
+
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Document
+import com.fleeksoft.ksoup.nodes.Element
+import com.fleeksoft.ksoup.nodes.Node
+import com.fleeksoft.ksoup.nodes.TextNode
+import net.sergeych.lyng.ModuleScope
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.ImportManager
+import net.sergeych.lyngio.stdlib_included.html_parseLyng
+
+private const val MODULE = "lyng.io.html.parse"
+
+fun createHtmlParseModule(scope: Scope): Boolean = createHtmlParseModule(scope.importManager)
+
+fun createHtmlParseModule(manager: ImportManager): Boolean {
+    if (MODULE in manager.packageNames) return false
+    manager.addPackage(MODULE) { module -> buildHtmlParseModule(module) }
+    return true
+}
+
+private suspend fun buildHtmlParseModule(module: ModuleScope) {
+    module.eval(Source(MODULE, html_parseLyng))
+    val html = ObjClass("Html").apply {
+        addClassFn("parse") {
+            val source = requiredArg<ObjString>(0).value
+            val baseUrl = (args.list.getOrNull(1) as? ObjString)?.value.orEmpty()
+            ObjHtmlDocument(Ksoup.parse(source, baseUrl))
+        }
+    }
+    module.addConst("Html", html)
+    module.addConst("HtmlDocument", ObjHtmlDocument.type)
+    module.addConst("HtmlElement", ObjHtmlElement.type)
+}
+
+private class ObjHtmlDocument(val document: Document) : Obj() {
+    override val objClass get() = type
+    companion object {
+        val type = object : ObjClass("HtmlDocument") {
+            override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use Html.parse()")
+        }.apply {
+            addProperty("title", getter = { ObjString(thisAs<ObjHtmlDocument>().document.title()) })
+            addFn("text") { ObjString(readableText(thisAs<ObjHtmlDocument>().document.body())) }
+            addFn("select") {
+                val css = requiredArg<ObjString>(0).value
+                ObjList(thisAs<ObjHtmlDocument>().document.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList())
+            }
+        }
+    }
+}
+
+private class ObjHtmlElement(val element: Element) : Obj() {
+    override val objClass get() = type
+    companion object {
+        val type = object : ObjClass("HtmlElement") {
+            override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use document.select()")
+        }.apply {
+            addFn("text") { ObjString(readableText(thisAs<ObjHtmlElement>().element)) }
+            addFn("attr") { ObjString(thisAs<ObjHtmlElement>().element.attr(requiredArg<ObjString>(0).value)) }
+            addFn("absUrl") { ObjString(thisAs<ObjHtmlElement>().element.absUrl(requiredArg<ObjString>(0).value)) }
+            addFn("select") {
+                val css = requiredArg<ObjString>(0).value
+                ObjList(thisAs<ObjHtmlElement>().element.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList())
+            }
+        }
+    }
+}
+
+private val excludedTags = setOf("script", "style", "template", "noscript")
+private val inlineWhitespace = Regex("[\\s\\u00a0]+")
+
+/** Iterative traversal avoids overflowing the host stack on deeply nested input. */
+private fun readableText(root: Element): String {
+    val text = StringBuilder()
+    val pending = ArrayDeque<Pair<Node, Boolean>>()
+    pending.addLast(root to false)
+    while (pending.isNotEmpty()) {
+        val (node, exit) = pending.removeLast()
+        when (node) {
+            is TextNode -> text.append(node.getWholeText().replace(inlineWhitespace, " "))
+            is Element -> {
+                if (node.normalName() in excludedTags || node.hasAttr("hidden")) continue
+                val block = node.isBlock() || node.normalName() == "br"
+                if (block) text.append('\n')
+                if (!exit) {
+                    if (block) pending.addLast(node to true)
+                    for (child in node.childNodes().asReversed()) pending.addLast(child to false)
+                }
+            }
+        }
+    }
+    return text.toString().lineSequence().map { it.replace(inlineWhitespace, " ").trim() }.filter { it.isNotEmpty() }.joinToString("\n")
+}

lyngio/src/commonTest/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModuleTest.kt

@@ -0,0 +1,59 @@
+package net.sergeych.lyng.io.html
+
+import kotlinx.coroutines.test.runTest
+import net.sergeych.lyng.Compiler
+import net.sergeych.lyng.Script
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.pacman.ImportManager
+import kotlin.test.*
+
+class LyngHtmlParseModuleTest {
+    private suspend fun execute(source: String) {
+        val scope = Script.newScope()
+        createHtmlParseModule(scope)
+        Compiler.compile(Source("html-parse-test", source.trimIndent()), scope.importManager).execute(scope)
+    }
+
+    @Test fun registrationIsIdempotent() {
+        val manager = ImportManager()
+        assertTrue(createHtmlParseModule(manager))
+        assertFalse(createHtmlParseModule(manager))
+    }
+
+    @Test fun parsesMalformedHtmlAndExtractsTextWithoutChangingDom() = runTest {
+        execute("""
+            import lyng.io.html.parse
+            val doc = Html.parse("<title>A &amp; B</title><style>bad css</style><body><h1>Heading</h1><p>Hello <b>world</b> &amp; &#65;<br>Next<p>Last<script>bad script</script><div hidden>secret</div><template>bad template</template><noscript>bad fallback</noscript>")
+            assert(doc.title == "A & B")
+            assert(doc.text() == "Heading\nHello world & A\nNext\nLast")
+            assert(doc.select("p").size == 2)
+            assert(doc.select("script").size == 1)
+            assert(doc.select("#missing").size == 0)
+            assert(doc.text() == doc.text())
+        """)
+    }
+
+    @Test fun selectorsAttributesBaseAndNestedSelection() = runTest {
+        execute("""
+            import lyng.io.html.parse
+            val doc = Html.parse("<base href='/docs/'><article><a class='ref' href='intro?a=1&amp;b=2'>Read &amp; learn</a></article>", "https://example.com/start")
+            val articles = doc.select("article")
+            val article = articles[0] as HtmlElement
+            val links = article.select("a.ref[href]")
+            val link = links[0] as HtmlElement
+            assert(link.text() == "Read & learn")
+            assert(link.attr("href") == "intro?a=1&b=2")
+            assert(link.absUrl("href") == "https://example.com/docs/intro?a=1&b=2")
+            assert(link.attr("missing") == "")
+            assert(link.absUrl("missing") == "")
+            val fragment = Html.parse("<p>One</p><p>Two</p>")
+            assert(fragment.text() == "One\nTwo")
+            assert(fragment.title == "")
+            assert(Html.parse("").text() == "")
+        """)
+    }
+
+    @Test fun badSelectorFailsClearly() = runTest {
+        assertFails { execute("import lyng.io.html.parse\nHtml.parse(\"<p>Hi\").select(\"[\")") }
+    }
+}

lyngio/stdlib/lyng/io/html_parse.lyng

@@ -0,0 +1,31 @@
+package lyng.io.html.parse
+
+/* Offline HTML5 parsing. Never fetches resources or executes JavaScript. */
+extern object Html {
+    /* Parse a document or fragment; baseUrl resolves relative attributes (no request). */
+    fun parse(source: String, baseUrl: String = ""): HtmlDocument
+}
+
+/* Parsed document. Retains the original DOM for selection and attributes. */
+extern class HtmlDocument {
+    /* Decoded document title, or an empty string. */
+    val title: String
+    /* Readable body text: entities decoded, block/BR boundaries preserved, whitespace normalized.
+       Excludes script, style, template, noscript and [hidden]; does not evaluate CSS visibility. */
+    fun text(): String
+    /* CSS selector; empty list when nothing matches. Invalid selectors raise an error. */
+    fun select(css: String): List<HtmlElement>
+}
+
+/* An element selected from a document. */
+extern class HtmlElement {
+    /* Same readable-text rules as HtmlDocument.text(), scoped to this element. */
+    fun text(): String
+    /* Decoded attribute value; empty string when absent. */
+    fun attr(name: String): String
+    /* Attribute resolved against the base URL (including HTML <base>); empty if unresolvable.
+       Inspect the scheme before using the URL: it is not restricted to HTTP/HTTPS. */
+    fun absUrl(name: String): String
+    /* CSS selector scoped to this element. */
+    fun select(css: String): List<HtmlElement>
+}