Add offline HTML parsing module to LyngIO

lyngio/build.gradle.kts

@@ -29,7 +29,7 @@ plugins {
 }
 
 group = "net.sergeych"
-version = providers.gradleProperty("lyngio.version").getOrElse("0.0.1-SNAPSHOT")
+version = providers.gradleProperty("lyngio.version").getOrElse("0.0.3-SNAPSHOT")
 
 private fun Project.sqliteLinuxLinkerOpts(vararg defaultDirs: String): List<String> {
     val overrideDir = providers.gradleProperty("sqlite3.lib.dir").orNull
@@ -141,6 +141,7 @@ kotlin {
                 api(libs.kotlinx.coroutines.core)
                 api(libs.mordant.core)
                 api(libs.ktor.client.core)
+                implementation("com.fleeksoft.ksoup:ksoup:0.2.6")
                 implementation(libs.ktor.client.websockets)
             }
         }
@@ -355,3 +356,9 @@ publishing {
         }
     }
 }
+
+// Lyng's JVM dependencies include Java 21 bytecode; keep the Java 17 library target.
+val testToolchains = extensions.getByType<org.gradle.jvm.toolchain.JavaToolchainService>()
+tasks.withType<Test>().configureEach {
+    javaLauncher.set(testToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(21)) })
+}

lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModule.kt

@@ -0,0 +1,96 @@
+package net.sergeych.lyng.io.html
+
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Document
+import com.fleeksoft.ksoup.nodes.Element
+import com.fleeksoft.ksoup.nodes.Node
+import com.fleeksoft.ksoup.nodes.TextNode
+import net.sergeych.lyng.ModuleScope
+import net.sergeych.lyng.Scope
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.obj.*
+import net.sergeych.lyng.pacman.ImportManager
+import net.sergeych.lyngio.stdlib_included.html_parseLyng
+
+private const val MODULE = "lyng.io.html.parse"
+
+fun createHtmlParseModule(scope: Scope): Boolean = createHtmlParseModule(scope.importManager)
+
+fun createHtmlParseModule(manager: ImportManager): Boolean {
+    if (MODULE in manager.packageNames) return false
+    manager.addPackage(MODULE) { module -> buildHtmlParseModule(module) }
+    return true
+}
+
+private suspend fun buildHtmlParseModule(module: ModuleScope) {
+    module.eval(Source(MODULE, html_parseLyng))
+    val html = ObjClass("Html").apply {
+        addClassFn("parse") {
+            val source = requiredArg<ObjString>(0).value
+            val baseUrl = (args.list.getOrNull(1) as? ObjString)?.value.orEmpty()
+            ObjHtmlDocument(Ksoup.parse(source, baseUrl))
+        }
+    }
+    module.addConst("Html", html)
+    module.addConst("HtmlDocument", ObjHtmlDocument.type)
+    module.addConst("HtmlElement", ObjHtmlElement.type)
+}
+
+private class ObjHtmlDocument(val document: Document) : Obj() {
+    override val objClass get() = type
+    companion object {
+        val type = object : ObjClass("HtmlDocument") {
+            override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use Html.parse()")
+        }.apply {
+            addProperty("title", getter = { ObjString(thisAs<ObjHtmlDocument>().document.title()) })
+            addFn("text") { ObjString(readableText(thisAs<ObjHtmlDocument>().document.body())) }
+            addFn("select") {
+                val css = requiredArg<ObjString>(0).value
+                ObjList(thisAs<ObjHtmlDocument>().document.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList())
+            }
+        }
+    }
+}
+
+private class ObjHtmlElement(val element: Element) : Obj() {
+    override val objClass get() = type
+    companion object {
+        val type = object : ObjClass("HtmlElement") {
+            override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use document.select()")
+        }.apply {
+            addFn("text") { ObjString(readableText(thisAs<ObjHtmlElement>().element)) }
+            addFn("attr") { ObjString(thisAs<ObjHtmlElement>().element.attr(requiredArg<ObjString>(0).value)) }
+            addFn("absUrl") { ObjString(thisAs<ObjHtmlElement>().element.absUrl(requiredArg<ObjString>(0).value)) }
+            addFn("select") {
+                val css = requiredArg<ObjString>(0).value
+                ObjList(thisAs<ObjHtmlElement>().element.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList())
+            }
+        }
+    }
+}
+
+private val excludedTags = setOf("script", "style", "template", "noscript")
+private val inlineWhitespace = Regex("[\\s\\u00a0]+")
+
+/** Iterative traversal avoids overflowing the host stack on deeply nested input. */
+private fun readableText(root: Element): String {
+    val text = StringBuilder()
+    val pending = ArrayDeque<Pair<Node, Boolean>>()
+    pending.addLast(root to false)
+    while (pending.isNotEmpty()) {
+        val (node, exit) = pending.removeLast()
+        when (node) {
+            is TextNode -> text.append(node.getWholeText().replace(inlineWhitespace, " "))
+            is Element -> {
+                if (node.normalName() in excludedTags || node.hasAttr("hidden")) continue
+                val block = node.isBlock() || node.normalName() == "br"
+                if (block) text.append('\n')
+                if (!exit) {
+                    if (block) pending.addLast(node to true)
+                    for (child in node.childNodes().asReversed()) pending.addLast(child to false)
+                }
+            }
+        }
+    }
+    return text.toString().lineSequence().map { it.replace(inlineWhitespace, " ").trim() }.filter { it.isNotEmpty() }.joinToString("\n")
+}

lyngio/src/commonTest/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModuleTest.kt

@@ -0,0 +1,59 @@
+package net.sergeych.lyng.io.html
+
+import kotlinx.coroutines.test.runTest
+import net.sergeych.lyng.Compiler
+import net.sergeych.lyng.Script
+import net.sergeych.lyng.Source
+import net.sergeych.lyng.pacman.ImportManager
+import kotlin.test.*
+
+class LyngHtmlParseModuleTest {
+    private suspend fun execute(source: String) {
+        val scope = Script.newScope()
+        createHtmlParseModule(scope)
+        Compiler.compile(Source("html-parse-test", source.trimIndent()), scope.importManager).execute(scope)
+    }
+
+    @Test fun registrationIsIdempotent() {
+        val manager = ImportManager()
+        assertTrue(createHtmlParseModule(manager))
+        assertFalse(createHtmlParseModule(manager))
+    }
+
+    @Test fun parsesMalformedHtmlAndExtractsTextWithoutChangingDom() = runTest {
+        execute("""
+            import lyng.io.html.parse
+            val doc = Html.parse("<title>A &amp; B</title><style>bad css</style><body><h1>Heading</h1><p>Hello <b>world</b> &amp; &#65;<br>Next<p>Last<script>bad script</script><div hidden>secret</div><template>bad template</template><noscript>bad fallback</noscript>")
+            assert(doc.title == "A & B")
+            assert(doc.text() == "Heading\nHello world & A\nNext\nLast")
+            assert(doc.select("p").size == 2)
+            assert(doc.select("script").size == 1)
+            assert(doc.select("#missing").size == 0)
+            assert(doc.text() == doc.text())
+        """)
+    }
+
+    @Test fun selectorsAttributesBaseAndNestedSelection() = runTest {
+        execute("""
+            import lyng.io.html.parse
+            val doc = Html.parse("<base href='/docs/'><article><a class='ref' href='intro?a=1&amp;b=2'>Read &amp; learn</a></article>", "https://example.com/start")
+            val articles = doc.select("article")
+            val article = articles[0] as HtmlElement
+            val links = article.select("a.ref[href]")
+            val link = links[0] as HtmlElement
+            assert(link.text() == "Read & learn")
+            assert(link.attr("href") == "intro?a=1&b=2")
+            assert(link.absUrl("href") == "https://example.com/docs/intro?a=1&b=2")
+            assert(link.attr("missing") == "")
+            assert(link.absUrl("missing") == "")
+            val fragment = Html.parse("<p>One</p><p>Two</p>")
+            assert(fragment.text() == "One\nTwo")
+            assert(fragment.title == "")
+            assert(Html.parse("").text() == "")
+        """)
+    }
+
+    @Test fun badSelectorFailsClearly() = runTest {
+        assertFails { execute("import lyng.io.html.parse\nHtml.parse(\"<p>Hi\").select(\"[\")") }
+    }
+}

lyngio/stdlib/lyng/io/html_parse.lyng

@@ -0,0 +1,31 @@
+package lyng.io.html.parse
+
+/* Offline HTML5 parsing. Never fetches resources or executes JavaScript. */
+extern object Html {
+    /* Parse a document or fragment; baseUrl resolves relative attributes (no request). */
+    fun parse(source: String, baseUrl: String = ""): HtmlDocument
+}
+
+/* Parsed document. Retains the original DOM for selection and attributes. */
+extern class HtmlDocument {
+    /* Decoded document title, or an empty string. */
+    val title: String
+    /* Readable body text: entities decoded, block/BR boundaries preserved, whitespace normalized.
+       Excludes script, style, template, noscript and [hidden]; does not evaluate CSS visibility. */
+    fun text(): String
+    /* CSS selector; empty list when nothing matches. Invalid selectors raise an error. */
+    fun select(css: String): List<HtmlElement>
+}
+
+/* An element selected from a document. */
+extern class HtmlElement {
+    /* Same readable-text rules as HtmlDocument.text(), scoped to this element. */
+    fun text(): String
+    /* Decoded attribute value; empty string when absent. */
+    fun attr(name: String): String
+    /* Attribute resolved against the base URL (including HTML <base>); empty if unresolvable.
+       Inspect the scheme before using the URL: it is not restricted to HTTP/HTTPS. */
+    fun absUrl(name: String): String
+    /* CSS selector scoped to this element. */
+    fun select(css: String): List<HtmlElement>
+}