diff --git a/.gitignore b/.gitignore index d2b04b1..245f251 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ test_output*.txt /bugs/ contents.db /docs/idea_plugin.md.deploy_backup +/.claude/settings.local.json diff --git a/README.md b/README.md index 4a408e1..bd81b9a 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ assertEquals(A.E.One, A.One) - [SQL Databases (lyng.io.db)](docs/lyng.io.db.md) - [HTTP Server (lyng.io.http.server)](docs/lyng.io.http.server.md) - [HTML Builder (lyng.io.html)](docs/lyng.io.html.md) +- [HTML Parsing (lyng.io.html.parse)](docs/lyng.io.html.parse.md) - [Process Execution (lyng.io.process)](docs/lyng.io.process.md) - [Time and Calendar Types](docs/time.md) - [Json support](docs/json_and_kotlin_serialization.md) diff --git a/docs/ai_stdlib_reference.md b/docs/ai_stdlib_reference.md index 9a79318..f02cf1d 100644 --- a/docs/ai_stdlib_reference.md +++ b/docs/ai_stdlib_reference.md @@ -98,6 +98,7 @@ Requires installing `lyngio` into the import manager from host code. - `import lyng.io.ws` (WebSocket client API; currently supported on JVM, capability-gated elsewhere) - `import lyng.io.net` (TCP/UDP transport API; currently supported on JVM, capability-gated elsewhere) - `import lyng.io.html` (pure Lyng HTML builder DSL: `html { body { h3 { +"text" } } }`) +- `import lyng.io.html.parse` (offline HTML5 parsing: `Html.parse(source, baseUrl = "")`, document title/text, CSS selectors, element text/attributes/absolute URLs; see [HTML parsing](lyng.io.html.parse.md)) - Shared network value-type packages are also available when installed by host code: - `import lyng.io.http.types` (`HttpHeaders`) - `import lyng.io.ws.types` (`WsMessage`) diff --git a/docs/lyng.io.html.md b/docs/lyng.io.html.md index 967e513..0f1c9ca 100644 --- a/docs/lyng.io.html.md +++ b/docs/lyng.io.html.md @@ -1,5 +1,7 @@ # lyng.io.html +For reading existing HTML, see [lyng.io.html.parse](lyng.io.html.parse.md). + `lyng.io.html` provides a pure Lyng HTML builder DSL. It uses Lyng context receiver extensions, so text can be appended with `+"text"` inside tag blocks without global builder state. diff --git a/docs/lyng.io.html.parse.md b/docs/lyng.io.html.parse.md new file mode 100644 index 0000000..029df20 --- /dev/null +++ b/docs/lyng.io.html.parse.md @@ -0,0 +1,41 @@ +# lyng.io.html.parse + +Offline HTML5 parsing backed by the core [Ksoup](https://github.com/fleeksoft/ksoup) +parser (no network extensions). Installed from `lyngio`: + +```kotlin +createHtmlParseModule(scope.importManager) +``` + +```lyng +import lyng.io.html.parse + +val doc = Html.parse("News

Hello & welcome

Next
", "https://example.com/") +println(doc.title) +println(doc.text()) +for (raw in doc.select("a[href]")) { + val link = raw as HtmlElement + println(link.text() + " => " + link.absUrl("href")) +} +``` + +`Html.parse(source, baseUrl = "")` accepts documents, fragments and malformed HTML. +It returns `HtmlDocument`, with `title: String`, `text(): String` and +`select(css): List`. Elements have `text()`, `attr(name)`, +`absUrl(name)` and `select(css)`. Missing selections return an empty list; +missing attributes or unresolvable absolute URLs return an empty string. +Invalid CSS selectors raise an error. `` participates in URL resolution. +Use the final fetched URL as the base when available. + +Text extraction decodes entities, normalizes whitespace, preserves block and +`br` boundaries as newlines, and excludes `script`, `style`, `template`, `noscript` +and `[hidden]` subtrees. Selection retains the original DOM. This is readable +text extraction, not CSS rendering or article detection; navigation and cookie +banners may remain. It does not preserve preformatted whitespace. + +Parsing never fetches resources or executes JavaScript. `absUrl` only resolves +URLs; it does not filter schemes. Fetch through the host's authorized HTTP API, +check status and content type, and choose bounded excerpts to return to an AI. +This API parses HTML only; XML parsing is separate future work. + +Canonical declarations: `lyngio/stdlib/lyng/io/html_parse.lyng`. diff --git a/lyngio/build.gradle.kts b/lyngio/build.gradle.kts index d338a99..afcda74 100644 --- a/lyngio/build.gradle.kts +++ b/lyngio/build.gradle.kts @@ -29,7 +29,7 @@ plugins { } group = "net.sergeych" -version = providers.gradleProperty("lyngio.version").getOrElse("0.0.1-SNAPSHOT") +version = providers.gradleProperty("lyngio.version").getOrElse("0.0.3-SNAPSHOT") private fun Project.sqliteLinuxLinkerOpts(vararg defaultDirs: String): List { val overrideDir = providers.gradleProperty("sqlite3.lib.dir").orNull @@ -141,6 +141,7 @@ kotlin { api(libs.kotlinx.coroutines.core) api(libs.mordant.core) api(libs.ktor.client.core) + implementation("com.fleeksoft.ksoup:ksoup:0.2.6") implementation(libs.ktor.client.websockets) } } @@ -355,3 +356,9 @@ publishing { } } } + +// Lyng's JVM dependencies include Java 21 bytecode; keep the Java 17 library target. +val testToolchains = extensions.getByType() +tasks.withType().configureEach { + javaLauncher.set(testToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(21)) }) +} diff --git a/lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModule.kt b/lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModule.kt new file mode 100644 index 0000000..bba3899 --- /dev/null +++ b/lyngio/src/commonMain/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModule.kt @@ -0,0 +1,96 @@ +package net.sergeych.lyng.io.html + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.nodes.Document +import com.fleeksoft.ksoup.nodes.Element +import com.fleeksoft.ksoup.nodes.Node +import com.fleeksoft.ksoup.nodes.TextNode +import net.sergeych.lyng.ModuleScope +import net.sergeych.lyng.Scope +import net.sergeych.lyng.Source +import net.sergeych.lyng.obj.* +import net.sergeych.lyng.pacman.ImportManager +import net.sergeych.lyngio.stdlib_included.html_parseLyng + +private const val MODULE = "lyng.io.html.parse" + +fun createHtmlParseModule(scope: Scope): Boolean = createHtmlParseModule(scope.importManager) + +fun createHtmlParseModule(manager: ImportManager): Boolean { + if (MODULE in manager.packageNames) return false + manager.addPackage(MODULE) { module -> buildHtmlParseModule(module) } + return true +} + +private suspend fun buildHtmlParseModule(module: ModuleScope) { + module.eval(Source(MODULE, html_parseLyng)) + val html = ObjClass("Html").apply { + addClassFn("parse") { + val source = requiredArg(0).value + val baseUrl = (args.list.getOrNull(1) as? ObjString)?.value.orEmpty() + ObjHtmlDocument(Ksoup.parse(source, baseUrl)) + } + } + module.addConst("Html", html) + module.addConst("HtmlDocument", ObjHtmlDocument.type) + module.addConst("HtmlElement", ObjHtmlElement.type) +} + +private class ObjHtmlDocument(val document: Document) : Obj() { + override val objClass get() = type + companion object { + val type = object : ObjClass("HtmlDocument") { + override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use Html.parse()") + }.apply { + addProperty("title", getter = { ObjString(thisAs().document.title()) }) + addFn("text") { ObjString(readableText(thisAs().document.body())) } + addFn("select") { + val css = requiredArg(0).value + ObjList(thisAs().document.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList()) + } + } + } +} + +private class ObjHtmlElement(val element: Element) : Obj() { + override val objClass get() = type + companion object { + val type = object : ObjClass("HtmlElement") { + override suspend fun callOn(scope: Scope): Obj = scope.raiseError("Use document.select()") + }.apply { + addFn("text") { ObjString(readableText(thisAs().element)) } + addFn("attr") { ObjString(thisAs().element.attr(requiredArg(0).value)) } + addFn("absUrl") { ObjString(thisAs().element.absUrl(requiredArg(0).value)) } + addFn("select") { + val css = requiredArg(0).value + ObjList(thisAs().element.select(css).map { ObjHtmlElement(it) as Obj }.toMutableList()) + } + } + } +} + +private val excludedTags = setOf("script", "style", "template", "noscript") +private val inlineWhitespace = Regex("[\\s\\u00a0]+") + +/** Iterative traversal avoids overflowing the host stack on deeply nested input. */ +private fun readableText(root: Element): String { + val text = StringBuilder() + val pending = ArrayDeque>() + pending.addLast(root to false) + while (pending.isNotEmpty()) { + val (node, exit) = pending.removeLast() + when (node) { + is TextNode -> text.append(node.getWholeText().replace(inlineWhitespace, " ")) + is Element -> { + if (node.normalName() in excludedTags || node.hasAttr("hidden")) continue + val block = node.isBlock() || node.normalName() == "br" + if (block) text.append('\n') + if (!exit) { + if (block) pending.addLast(node to true) + for (child in node.childNodes().asReversed()) pending.addLast(child to false) + } + } + } + } + return text.toString().lineSequence().map { it.replace(inlineWhitespace, " ").trim() }.filter { it.isNotEmpty() }.joinToString("\n") +} diff --git a/lyngio/src/commonTest/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModuleTest.kt b/lyngio/src/commonTest/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModuleTest.kt new file mode 100644 index 0000000..0d1a764 --- /dev/null +++ b/lyngio/src/commonTest/kotlin/net/sergeych/lyng/io/html/LyngHtmlParseModuleTest.kt @@ -0,0 +1,59 @@ +package net.sergeych.lyng.io.html + +import kotlinx.coroutines.test.runTest +import net.sergeych.lyng.Compiler +import net.sergeych.lyng.Script +import net.sergeych.lyng.Source +import net.sergeych.lyng.pacman.ImportManager +import kotlin.test.* + +class LyngHtmlParseModuleTest { + private suspend fun execute(source: String) { + val scope = Script.newScope() + createHtmlParseModule(scope) + Compiler.compile(Source("html-parse-test", source.trimIndent()), scope.importManager).execute(scope) + } + + @Test fun registrationIsIdempotent() { + val manager = ImportManager() + assertTrue(createHtmlParseModule(manager)) + assertFalse(createHtmlParseModule(manager)) + } + + @Test fun parsesMalformedHtmlAndExtractsTextWithoutChangingDom() = runTest { + execute(""" + import lyng.io.html.parse + val doc = Html.parse("A & B

Heading

Hello world & A
Next

Last

") + assert(doc.title == "A & B") + assert(doc.text() == "Heading\nHello world & A\nNext\nLast") + assert(doc.select("p").size == 2) + assert(doc.select("script").size == 1) + assert(doc.select("#missing").size == 0) + assert(doc.text() == doc.text()) + """) + } + + @Test fun selectorsAttributesBaseAndNestedSelection() = runTest { + execute(""" + import lyng.io.html.parse + val doc = Html.parse("", "https://example.com/start") + val articles = doc.select("article") + val article = articles[0] as HtmlElement + val links = article.select("a.ref[href]") + val link = links[0] as HtmlElement + assert(link.text() == "Read & learn") + assert(link.attr("href") == "intro?a=1&b=2") + assert(link.absUrl("href") == "https://example.com/docs/intro?a=1&b=2") + assert(link.attr("missing") == "") + assert(link.absUrl("missing") == "") + val fragment = Html.parse("

One

Two

") + assert(fragment.text() == "One\nTwo") + assert(fragment.title == "") + assert(Html.parse("").text() == "") + """) + } + + @Test fun badSelectorFailsClearly() = runTest { + assertFails { execute("import lyng.io.html.parse\nHtml.parse(\"

Hi\").select(\"[\")") } + } +} diff --git a/lyngio/stdlib/lyng/io/html_parse.lyng b/lyngio/stdlib/lyng/io/html_parse.lyng new file mode 100644 index 0000000..3b93bb4 --- /dev/null +++ b/lyngio/stdlib/lyng/io/html_parse.lyng @@ -0,0 +1,31 @@ +package lyng.io.html.parse + +/* Offline HTML5 parsing. Never fetches resources or executes JavaScript. */ +extern object Html { + /* Parse a document or fragment; baseUrl resolves relative attributes (no request). */ + fun parse(source: String, baseUrl: String = ""): HtmlDocument +} + +/* Parsed document. Retains the original DOM for selection and attributes. */ +extern class HtmlDocument { + /* Decoded document title, or an empty string. */ + val title: String + /* Readable body text: entities decoded, block/BR boundaries preserved, whitespace normalized. + Excludes script, style, template, noscript and [hidden]; does not evaluate CSS visibility. */ + fun text(): String + /* CSS selector; empty list when nothing matches. Invalid selectors raise an error. */ + fun select(css: String): List +} + +/* An element selected from a document. */ +extern class HtmlElement { + /* Same readable-text rules as HtmlDocument.text(), scoped to this element. */ + fun text(): String + /* Decoded attribute value; empty string when absent. */ + fun attr(name: String): String + /* Attribute resolved against the base URL (including HTML ); empty if unresolvable. + Inspect the scheme before using the URL: it is not restricted to HTTP/HTTPS. */ + fun absUrl(name: String): String + /* CSS selector scoped to this element. */ + fun select(css: String): List +}