diff --git a/.gitignore b/.gitignore index 05436bd..83b4611 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ node_modules/ /composeApp/release/ /test_books/ /.junie/ +/.local/ diff --git a/README.md b/README.md index 3ab7392..9c1b9af 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ in your IDE’s toolbar or build it directly from the terminal: ### Build and Run Desktop (JVM) Application +Desktop offline English/Russian speech: [CPU TTS setup and audition](docs/desktop-tts.md). + To build and run the development version of the desktop app, use the run configuration from the run widget in your IDE’s toolbar or run it directly from the terminal: @@ -89,4 +91,4 @@ Learn more about [Kotlin Multiplatform](https://www.jetbrains.com/help/kotlin-mu We would appreciate your feedback on Compose/Web and Kotlin/Wasm in the public Slack channel [#compose-web](https://slack-chats.kotlinlang.org/c/compose-web). -If you face any issues, please report them on [YouTrack](https://youtrack.jetbrains.com/newIssue?project=CMP). \ No newline at end of file +If you face any issues, please report them on [YouTrack](https://youtrack.jetbrains.com/newIssue?project=CMP). diff --git a/composeApp/build.gradle.kts b/composeApp/build.gradle.kts index 49fafcd..3ea49bf 100644 --- a/composeApp/build.gradle.kts +++ b/composeApp/build.gradle.kts @@ -96,10 +96,29 @@ kotlin { jvmMain.dependencies { implementation(compose.desktop.currentOs) implementation(libs.kotlinx.coroutinesSwing) + implementation(libs.onnxruntime) + implementation(libs.jackson.databind) + } + jvmTest.dependencies { + implementation(libs.kotlinx.coroutinesTest) } } } +tasks.register("ttsSample") { + group = "verification" + description = "Generate plain English/Russian WAV samples using the local Supertonic CPU model." + dependsOn("jvmMainClasses") + mainClass.set("net.sergeych.toread.tts.TtsSampleKt") + classpath = configurations["jvmRuntimeClasspath"] + kotlin.targets["jvm"].compilations["main"].output.allOutputs + workingDir = rootProject.projectDir + args(layout.buildDirectory.dir("tts-samples").get().asFile.absolutePath) + if (providers.gradleProperty("ttsPlay").orNull == "true") args("--play") + if (providers.gradleProperty("ttsStress").orNull == "true") args("--stress") + if (providers.gradleProperty("ttsRuAccent").orNull == "true") args("--ruaccent") + if (providers.gradleProperty("ttsSilero").orNull == "true") args("--silero") +} + android { namespace = "net.sergeych.toread" compileSdk = libs.versions.android.compileSdk.get().toInt() diff --git a/composeApp/src/commonMain/kotlin/net/sergeych/toread/Localization.kt b/composeApp/src/commonMain/kotlin/net/sergeych/toread/Localization.kt index 40241be..457f21c 100644 --- a/composeApp/src/commonMain/kotlin/net/sergeych/toread/Localization.kt +++ b/composeApp/src/commonMain/kotlin/net/sergeych/toread/Localization.kt @@ -174,6 +174,9 @@ internal open class AppStrings { open val offlineVoice = "Offline voice" open val autoRussian = "Auto Russian" open val loadingVoices = "Loading voices..." + open val preparingSpeech = "Preparing speech…" + open val offlineSpeechModelMissing = "Download the offline speech model first; see the desktop setup guide." + open val speechPlaybackFailed = "Speech playback failed. Check the offline model and audio output, then try again." open val imageCopied = "Image copied" open val copyFailed = "Copy failed" @@ -399,6 +402,9 @@ internal object RussianStrings : AppStrings() { override val offlineVoice = "Офлайн-голос" override val autoRussian = "Авто: русский" override val loadingVoices = "Загружаем голоса..." + override val preparingSpeech = "Подготавливаем речь…" + override val offlineSpeechModelMissing = "Сначала загрузите офлайн-модель речи по инструкции для компьютера." + override val speechPlaybackFailed = "Не удалось воспроизвести речь. Проверьте офлайн-модель и вывод звука и попробуйте снова." override val imageCopied = "Изображение скопировано" override val copyFailed = "Не удалось скопировать" diff --git a/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReadAloudPlatform.kt b/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReadAloudPlatform.kt index 79e7c98..00f5dfb 100644 --- a/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReadAloudPlatform.kt +++ b/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReadAloudPlatform.kt @@ -17,6 +17,7 @@ data class ReadAloudState( val active: Boolean = false, val playing: Boolean = false, val sentenceIndex: Int? = null, + val buffering: Boolean = false, ) data class ReadAloudEngineOption( @@ -45,6 +46,7 @@ data class ReadAloudSettingsState( val textReplacements: List = emptyList(), val loading: Boolean = false, val message: String? = null, + val usesSystemDefaults: Boolean = true, ) expect object ReadAloudPlatform { diff --git a/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReaderScreen.kt b/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReaderScreen.kt index 98c043e..ef8db06 100644 --- a/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReaderScreen.kt +++ b/composeApp/src/commonMain/kotlin/net/sergeych/toread/ReaderScreen.kt @@ -204,6 +204,10 @@ internal fun BookView( } } + LaunchedEffect(readAloudSettings.message) { + readAloudSettings.message?.let { snackbarHostState.showSnackbar(it) } + } + fun openReaderLink(href: String) { if (href.isWebUrl()) { openPlatformUrl(href) @@ -549,6 +553,7 @@ internal fun BookView( if (readAloudPanelVisible && readAloudState.active) { ReadAloudPanel( playing = readAloudState.playing, + buffering = readAloudState.buffering, onPlayStop = { if (readAloudState.playing) { ReadAloudPlatform.stop() @@ -1626,6 +1631,7 @@ private val DarkReaderTextPalette = listOf( @Composable private fun ReadAloudPanel( playing: Boolean, + buffering: Boolean, onPlayStop: () -> Unit, onBack: () -> Unit, onForward: () -> Unit, @@ -1642,6 +1648,10 @@ private fun ReadAloudPanel( horizontalArrangement = Arrangement.Center, verticalAlignment = Alignment.CenterVertically, ) { + if (buffering) { + CircularProgressIndicator(modifier = Modifier.size(20.dp)) + Text(strings.preparingSpeech, modifier = Modifier.padding(horizontal = 8.dp)) + } IconButton(onClick = onBack) { Icon(Icons.Filled.Replay, contentDescription = strings.previousSentence) } @@ -1686,7 +1696,7 @@ private fun ReadAloudSettingsDialog( Text(selectedEngineLabel) } DropdownMenu(expanded = engineMenuOpen, onDismissRequest = { engineMenuOpen = false }) { - DropdownMenuItem( + if (state.usesSystemDefaults) DropdownMenuItem( text = { Text(strings.systemDefault) }, onClick = { engineMenuOpen = false @@ -1711,7 +1721,7 @@ private fun ReadAloudSettingsDialog( Text(selectedVoiceLabel) } DropdownMenu(expanded = voiceMenuOpen, onDismissRequest = { voiceMenuOpen = false }) { - DropdownMenuItem( + if (state.usesSystemDefaults) DropdownMenuItem( text = { Text(strings.autoRussian) }, onClick = { voiceMenuOpen = false diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/ReadAloudPlatform.jvm.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/ReadAloudPlatform.jvm.kt index 6dba933..fa07cb3 100644 --- a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/ReadAloudPlatform.jvm.kt +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/ReadAloudPlatform.jvm.kt @@ -1,20 +1,54 @@ package net.sergeych.toread -import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import net.sergeych.toread.tts.DesktopReadAloudPlayer +import net.sergeych.toread.tts.SpeechAudio +import net.sergeych.toread.tts.RuAccentProcess +import net.sergeych.toread.tts.SileroProcess +import net.sergeych.toread.tts.SupertonicTts +import net.sergeych.toread.tts.playDesktopAudio actual object ReadAloudPlatform { - actual val isSupported: Boolean = false - private val mutableState = MutableStateFlow(ReadAloudState()) - actual val state: StateFlow = mutableState - private val mutableSettingsState = MutableStateFlow(ReadAloudSettingsState()) - actual val settingsState: StateFlow = mutableSettingsState + actual val isSupported: Boolean = true + private val inferenceLock = Mutex() + private var synthesizer: SupertonicTts? = null + private val accentor = if (RuAccentProcess.isInstalled()) RuAccentProcess() else null + private val silero = if (SileroProcess.isInstalled()) SileroProcess() else null + private val player = DesktopReadAloudPlayer( + scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate), + synthesize = { text, voice -> + withContext(Dispatchers.IO) { + inferenceLock.withLock { + val context = currentCoroutineContext() + context.ensureActive() + val engine = synthesizer ?: SupertonicTts().also { synthesizer = it } + context.ensureActive() + SpeechAudio(engine.synthesize(text, voice, checkCancelled = { context.ensureActive() }), engine.sampleRate) + } + } + }, + playAudio = ::playDesktopAudio, + installed = { SupertonicTts.isInstalled() }, + accentuate = accentor?.let { processor -> { text -> processor.accentuate(text) } }, + silero = silero?.let { processor -> { text, voice -> processor.synthesize(text, voice) } }, + ) + actual val state: StateFlow = player.state + actual val settingsState: StateFlow = player.settingsState - actual fun prepare(fileId: String, bookTitle: String, sentences: List, startIndex: Int) = Unit - actual fun play() = Unit - actual fun stop() = Unit - actual fun skip(delta: Int) = Unit - actual fun refreshSettings() = Unit - actual fun selectEngine(engineId: String?) = Unit - actual fun selectVoice(voiceId: String?) = Unit + actual fun prepare(fileId: String, bookTitle: String, sentences: List, startIndex: Int) = + player.prepare(sentences, startIndex) + actual fun play() = player.play() + actual fun stop() = player.stop() + actual fun skip(delta: Int) = player.skip(delta) + actual fun refreshSettings() = player.refreshSettings() + actual fun selectEngine(engineId: String?) = player.selectEngine(engineId) + actual fun selectVoice(voiceId: String?) = player.selectVoice(voiceId) } diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopAudio.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopAudio.kt new file mode 100644 index 0000000..4875b31 --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopAudio.kt @@ -0,0 +1,70 @@ +package net.sergeych.toread.tts + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withContext +import java.io.ByteArrayInputStream +import java.io.File +import javax.sound.sampled.AudioFileFormat +import javax.sound.sampled.AudioFormat +import javax.sound.sampled.AudioInputStream +import javax.sound.sampled.AudioSystem +import javax.sound.sampled.LineEvent +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume + +internal fun SpeechAudio.pcm16(): ByteArray = ByteArray(samples.size * 2).also { bytes -> + samples.forEachIndexed { index, sample -> + val pcm = (sample.takeIf { it.isFinite() }?.coerceIn(-1f, 1f)?.times(32767) ?: 0f).toInt() + bytes[index * 2] = pcm.toByte() + bytes[index * 2 + 1] = (pcm shr 8).toByte() + } +} + +internal suspend fun playDesktopAudio(audio: SpeechAudio) = withContext(Dispatchers.IO) { + val format = AudioFormat(audio.sampleRate.toFloat(), 16, 1, true, false) + val pcm = audio.pcm16() + val clip = AudioSystem.getClip() + try { + currentCoroutineContext().ensureActive() + clip.open(format, pcm, 0, pcm.size) + suspendCancellableCoroutine { continuation -> + val completed = AtomicBoolean(false) + clip.addLineListener { event -> + if (event.type == LineEvent.Type.STOP && completed.compareAndSet(false, true)) { + continuation.resume(Unit) + } + } + continuation.invokeOnCancellation { + clip.stop() + clip.close() + } + if (continuation.isActive) clip.start() + } + } finally { + clip.stop() + clip.close() + } +} + +internal fun SpeechAudio.writeWav(file: File) { + file.parentFile?.mkdirs() + val format = AudioFormat(sampleRate.toFloat(), 16, 1, true, false) + AudioInputStream(ByteArrayInputStream(pcm16()), format, samples.size.toLong()).use { + AudioSystem.write(it, AudioFileFormat.Type.WAVE, file) + } +} + +internal fun joinSpeechAudio(parts: List): SpeechAudio { + require(parts.isNotEmpty()) { "No speech to play" } + if (parts.size == 1) return parts.single() + val rate = parts.first().sampleRate + require(parts.all { it.sampleRate == rate }) { "Mismatched speech sample rates" } + val pause = rate / 10 + val samples = FloatArray(parts.sumOf { it.samples.size } + pause * (parts.size - 1)) + var offset = 0 + parts.forEach { part -> part.samples.copyInto(samples, offset); offset += part.samples.size + pause } + return SpeechAudio(samples, rate) +} diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayer.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayer.kt new file mode 100644 index 0000000..ce5f446 --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayer.kt @@ -0,0 +1,158 @@ +package net.sergeych.toread.tts + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import net.sergeych.toread.ReadAloudEngineOption +import net.sergeych.toread.ReadAloudSentence +import net.sergeych.toread.ReadAloudSettingsState +import net.sergeych.toread.ReadAloudState +import net.sergeych.toread.ReadAloudVoiceOption +import net.sergeych.toread.strings + +internal data class SpeechAudio(val samples: FloatArray, val sampleRate: Int) + +/** Commands and state changes run on the UI dispatcher; inference/audio run off-thread. + * A playback job owns its lookahead, so stop/skip/voice changes cannot publish stale audio. + */ +internal class DesktopReadAloudPlayer( + private val scope: CoroutineScope, + private val synthesize: suspend (String, String) -> SpeechAudio, + private val playAudio: suspend (SpeechAudio) -> Unit, + private val installed: () -> Boolean, + private val accentuate: (suspend (String) -> String)? = null, + private val silero: (suspend (String, String) -> SpeechAudio)? = null, +) { + private fun voices(engine: String) = if (engine == "silero") + SileroProcess.voices.map { ReadAloudVoiceOption(it, it.replaceFirstChar(Char::uppercase), "ru (English: F1)", false) } + else SupertonicTts.voices.map { ReadAloudVoiceOption(it, it, "en / ru", false) } + private val defaultEngine = if (silero != null) "silero" else if (accentuate != null) "supertonic-3-ruaccent" else "supertonic-3" + private val mutableState = MutableStateFlow(ReadAloudState()) + val state = mutableState.asStateFlow() + private val mutableSettings = MutableStateFlow(ReadAloudSettingsState( + engines = (if (silero != null) listOf(ReadAloudEngineOption("silero", "Silero (RU) / Supertonic (EN)")) else emptyList()) + + listOf(ReadAloudEngineOption("supertonic-3", "Supertonic 3")) + + if (accentuate != null) listOf(ReadAloudEngineOption("supertonic-3-ruaccent", "Supertonic 3 + RUAccent")) else emptyList(), + selectedEngineId = defaultEngine, + voices = voices(defaultEngine), + selectedVoiceId = if (silero != null) "xenia" else "F1", + usesSystemDefaults = false, + )) + val settingsState = mutableSettings.asStateFlow() + private var queue = emptyList() + private var index = 0 + private var playback: Job? = null + + fun prepare(sentences: List, startIndex: Int) { + stop() + queue = sentences.toList() + index = if (queue.isEmpty()) 0 else startIndex.coerceIn(queue.indices) + mutableState.value = ReadAloudState(active = queue.isNotEmpty(), sentenceIndex = queue.getOrNull(index)?.index) + } + + fun play() { + if (queue.isEmpty()) return + playback?.cancel() + mutableSettings.value = mutableSettings.value.copy(message = null) + if (!installed()) { + mutableState.value = mutableState.value.copy(active = true, playing = false, buffering = false) + mutableSettings.value = mutableSettings.value.copy(message = strings.offlineSpeechModelMissing) + return + } + val voice = mutableSettings.value.selectedVoiceId ?: "F1" + val useSilero = mutableSettings.value.selectedEngineId == "silero" + val useAccents = mutableSettings.value.selectedEngineId == "supertonic-3-ruaccent" + suspend fun render(text: String): SpeechAudio { + if (useSilero) return if (SupertonicTts.languageOf(text) == "ru") silero!!.invoke(text, voice) + else synthesize(text, "F1") + val prepared = if (useAccents) accentuate!!.invoke(text) else text + // Accent marks increase character count; re-chunk before entering the TTS model. + return joinSpeechAudio(SupertonicTts.chunks(SupertonicTts.russianStressToUnicode(prepared)).map { synthesize(it, voice) }) + } + val chunks = queue.asSequence().withIndex().drop(index).flatMap { (position, sentence) -> + val parts = SupertonicTts.chunks(sentence.spokenText) + parts.asSequence().mapIndexed { chunkIndex, text -> + Chunk(position, sentence, text, chunkIndex == 0, chunkIndex == parts.lastIndex) + } + }.iterator() + mutableState.value = mutableState.value.copy(active = true, playing = true, buffering = true) + playback = scope.launch { + try { + coroutineScope { + // Keep at most one audio chunk ahead of the currently playing chunk. + var next = if (chunks.hasNext()) chunks.next() else null + var pending = next?.let { chunk -> async { render(chunk.text) } } + while (next != null) { + val current = next + mutableState.value = mutableState.value.copy(buffering = pending?.isCompleted != true) + val audio = pending!!.await() + next = if (chunks.hasNext()) chunks.next() else null + pending = next?.let { chunk -> async { render(chunk.text) } } + index = current.position + mutableState.value = ReadAloudState(active = true, playing = true, sentenceIndex = current.sentence.index) + if (current.first) delay(current.sentence.pauseBeforeMillis) + playAudio(audio) + if (current.last) delay(current.sentence.pauseAfterMillis) + } + } + mutableState.value = ReadAloudState() + } catch (cancelled: CancellationException) { + throw cancelled + } catch (error: Exception) { + System.err.println("Desktop TTS failed: ${error.javaClass.simpleName}: ${error.message}") + mutableState.value = mutableState.value.copy(playing = false, buffering = false) + mutableSettings.value = mutableSettings.value.copy(message = strings.speechPlaybackFailed) + } + } + } + + fun stop() { + playback?.cancel() + playback = null + mutableState.value = ReadAloudState() + } + + fun skip(delta: Int) { + if (queue.isEmpty()) return + val wasPlaying = mutableState.value.playing + playback?.cancel() + index = (index.toLong() + delta).coerceIn(0, queue.lastIndex.toLong()).toInt() + mutableState.value = ReadAloudState(active = true, sentenceIndex = queue[index].index) + if (wasPlaying) play() + } + + fun selectVoice(id: String?) { + val voice = id ?: if (mutableSettings.value.selectedEngineId == "silero") "xenia" else "F1" + if (mutableSettings.value.voices.none { it.id == voice } || voice == mutableSettings.value.selectedVoiceId) return + mutableSettings.value = mutableSettings.value.copy(selectedVoiceId = voice) + if (mutableState.value.playing) play() + } + + fun selectEngine(id: String?) { + val engine = id ?: defaultEngine + if (mutableSettings.value.engines.none { it.id == engine } || engine == mutableSettings.value.selectedEngineId) return + val options = voices(engine) + mutableSettings.value = mutableSettings.value.copy(selectedEngineId = engine, voices = options, + selectedVoiceId = mutableSettings.value.selectedVoiceId?.takeIf { id -> options.any { it.id == id } } + ?: if (engine == "silero") "xenia" else "F1") + if (mutableState.value.playing) play() + } + + fun refreshSettings() { + mutableSettings.value = mutableSettings.value.copy(message = if (installed()) null else strings.offlineSpeechModelMissing) + } + + private data class Chunk( + val position: Int, + val sentence: ReadAloudSentence, + val text: String, + val first: Boolean, + val last: Boolean, + ) +} diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/RuAccentProcess.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/RuAccentProcess.kt new file mode 100644 index 0000000..dee0afa --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/RuAccentProcess.kt @@ -0,0 +1,90 @@ +package net.sergeych.toread.tts + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.isActive +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeout +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.File + +/** Optional desktop-only CPU accentor. Persistent local process, no HTTP or remote text service. */ +internal class RuAccentProcess(private val directory: File = installationDirectory()) : AutoCloseable { + private val mapper = ObjectMapper() + private val lock = Mutex() + @Volatile private var worker: Worker? = null + + init { + Runtime.getRuntime().addShutdownHook(Thread { close() }) + } + + suspend fun accentuate(text: String): String { + // Respect explicitly supplied stress; English needs no Russian processor. + if (SupertonicTts.languageOf(text) == "en" || '\u0301' in text || Regex("\\+[аеёиоуыэюяАЕЁИОУЫЭЮЯ]").containsMatchIn(text)) return text + return lock.withLock { + try { + currentCoroutineContext().ensureActive() + val current = worker?.takeIf { it.process.isAlive } ?: startWorker().also { worker = it } + withTimeout(90_000) { + coroutineScope { + val response = async(Dispatchers.IO) { + if (!current.ready) { + val ready = current.output.readLine() ?: error("RuAccent exited while loading") + check(mapper.readTree(ready)["ready"]?.asBoolean() == true) { "RuAccent did not initialize" } + current.ready = true + } + current.input.write(mapper.writeValueAsString(mapOf("text" to text))) + current.input.newLine() + current.input.flush() + val line = current.output.readLine() ?: error("RuAccent exited during inference") + val result = mapper.readTree(line) + check(result["error"] == null) { "RuAccent inference failed" } + result["text"]?.asText()?.takeIf { it.isNotBlank() } ?: error("Empty RuAccent response") + } + try { + response.await() + } finally { + // A blocked pipe read cannot be interrupted reliably; killing the + // cancelled worker unblocks it before the next request acquires the lock. + if (!isActive) current.process.destroyForcibly() + } + } + } + } catch (error: Exception) { + close() + throw error + } + } + } + + private fun startWorker(): Worker { + val runtime = mapper.readTree(File(directory, "runtime.json")) + val process = ProcessBuilder(runtime["python"].asText(), "-u", File(directory, "worker.py").path, directory.path) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .apply { + environment()["HF_HUB_OFFLINE"] = "1" + environment()["TRANSFORMERS_OFFLINE"] = "1" + environment()["HF_HUB_DISABLE_TELEMETRY"] = "1" + }.start() + return Worker(process, process.inputStream.bufferedReader(Charsets.UTF_8), process.outputStream.bufferedWriter(Charsets.UTF_8)) + } + + override fun close() { + worker?.process?.destroyForcibly() + worker = null + } + + private class Worker(val process: Process, val output: BufferedReader, val input: BufferedWriter, var ready: Boolean = false) + + companion object { + fun installationDirectory(): File = System.getenv("YEREED_RUACCENT_HOME")?.let(::File) + ?: File(SupertonicTts.modelDirectory().parentFile, "ruaccent") + fun isInstalled(): Boolean = File(installationDirectory(), "runtime.json").isFile && File(installationDirectory(), "worker.py").isFile + } +} diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SileroProcess.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SileroProcess.kt new file mode 100644 index 0000000..94e8cfb --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SileroProcess.kt @@ -0,0 +1,98 @@ +package net.sergeych.toread.tts + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.isActive +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeout +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.Base64 + +/** Desktop-only CPU Silero synthesizer. Persistent local process, no HTTP or remote text service. */ +internal class SileroProcess(private val directory: File = installationDirectory()) : AutoCloseable { + private val mapper = ObjectMapper() + private val lock = Mutex() + @Volatile private var worker: Worker? = null + + init { + Runtime.getRuntime().addShutdownHook(Thread { close() }) + } + + suspend fun synthesize(text: String, voice: String): SpeechAudio { + return lock.withLock { + try { + currentCoroutineContext().ensureActive() + val current = worker?.takeIf { it.process.isAlive } ?: startWorker().also { worker = it } + withTimeout(90_000) { + coroutineScope { + val response = async(Dispatchers.IO) { + if (!current.ready) { + val ready = current.output.readLine() ?: error("Silero exited while loading") + check(mapper.readTree(ready)["ready"]?.asBoolean() == true) { "Silero did not initialize" } + current.ready = true + } + current.input.write(mapper.writeValueAsString(mapOf("text" to text, "voice" to voice))) + current.input.newLine() + current.input.flush() + val line = current.output.readLine() ?: error("Silero exited during inference") + val result = mapper.readTree(line) + check(result["error"] == null) { "Silero inference failed" } + val bytes = Base64.getDecoder().decode(result["audio"].asText()) + check(bytes.isNotEmpty() && bytes.size % 4 == 0) { "Invalid Silero audio" } + val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() + val samples = FloatArray(buffer.remaining()).also { buffer.get(it) } + check(samples.all { it.isFinite() }) { "Non-finite Silero audio" } + SpeechAudio(samples, result["sampleRate"].asInt().also { check(it == 48000) }) + } + try { + response.await() + } finally { + // A blocked pipe read cannot be interrupted reliably; killing the + // cancelled worker unblocks it before the next request acquires the lock. + if (!isActive) current.process.destroyForcibly() + } + } + } + } catch (error: Exception) { + close() + throw error + } + } + } + + private fun startWorker(): Worker { + val runtime = mapper.readTree(File(directory, "runtime.json")) + val script = SileroProcess::class.java.getResourceAsStream("/tts/silero-worker.py")!!.use { it.reader().readText() } + val process = ProcessBuilder(runtime["python"].asText(), "-u", "-c", script, directory.path) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .apply { + environment()["HF_HUB_OFFLINE"] = "1" + environment()["TRANSFORMERS_OFFLINE"] = "1" + environment()["HF_HUB_DISABLE_TELEMETRY"] = "1" + }.start() + return Worker(process, process.inputStream.bufferedReader(Charsets.UTF_8), process.outputStream.bufferedWriter(Charsets.UTF_8)) + } + + override fun close() { + worker?.process?.destroyForcibly() + worker = null + } + + private class Worker(val process: Process, val output: BufferedReader, val input: BufferedWriter, var ready: Boolean = false) + + companion object { + val voices = listOf("xenia", "baya", "aidar", "kseniya", "eugene") + fun installationDirectory(): File = System.getenv("YEREED_SILERO_HOME")?.let(::File) + ?: File(SupertonicTts.modelDirectory().parentFile, "silero-v5_5-ru") + fun isInstalled(): Boolean = File(installationDirectory(), "runtime.json").isFile && File(installationDirectory(), "model.pt").length() > 0 + } +} diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SupertonicTts.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SupertonicTts.kt new file mode 100644 index 0000000..229beba --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/SupertonicTts.kt @@ -0,0 +1,190 @@ +package net.sergeych.toread.tts + +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import ai.onnxruntime.OrtSession +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import java.io.File +import java.nio.FloatBuffer +import java.text.Normalizer +import java.util.Random + +/** CPU-only, single-utterance adaptation of Supertone's MIT Java example. + * See docs/licenses/supertonic-code-MIT.txt. Call from one worker at a time. + */ +internal class SupertonicTts(val directory: File = modelDirectory()) : AutoCloseable { + private val mapper = ObjectMapper() + private val environment = OrtEnvironment.getEnvironment() + private val config = mapper.readTree(File(directory, "onnx/tts.json")) + private val indexer = mapper.readTree(File(directory, "onnx/unicode_indexer.json")) + .map { it.asLong() }.toLongArray() + val sampleRate: Int = config["ae"]["sample_rate"].asInt() + private val compression = config["ttl"]["chunk_compress_factor"].asInt() + private val chunkSize = config["ae"]["base_chunk_size"].asInt() * compression + private val latentDimension = config["ttl"]["latent_dim"].asInt() * compression + private val sessions = mutableListOf() + private val styles = mutableMapOf() + + init { + try { + OrtSession.SessionOptions().use { options -> + // The standard (non-GPU) ONNX artifact uses CPUExecutionProvider only. + options.setIntraOpNumThreads(4.coerceAtMost(Runtime.getRuntime().availableProcessors())) + options.setInterOpNumThreads(1) + for (name in listOf("duration_predictor", "text_encoder", "vector_estimator", "vocoder")) { + sessions += environment.createSession(File(directory, "onnx/$name.onnx").path, options) + } + } + } catch (error: Throwable) { + close() + throw error + } + } + + fun synthesize( + text: String, + voice: String = "F1", + language: String = languageOf(text), + seed: Long? = null, + checkCancelled: () -> Unit = {}, + ): FloatArray { + require(voice in voices) { "Unknown voice: $voice" } + require(language in listOf("en", "ru", "na")) { "Unsupported language: $language" } + require(text.isNotBlank() && text.length <= MAX_CHARS) { "Expected 1–$MAX_CHARS characters" } + checkCancelled() + val prepared = prepareText(text, language) + // Unknown symbols become spaces, never invalid (-1) embedding indices. + val ids = prepared.codePoints().toArray().map { point -> + indexer.getOrNull(point)?.takeIf { it >= 0 } ?: indexer[' '.code] + }.toLongArray() + val style = styles.getOrPut(voice) { mapper.readTree(File(directory, "voice_styles/$voice.json")) } + NativeResources().use { resources -> + fun tensor(value: Any) = resources.keep(OnnxTensor.createTensor(environment, value)) + fun styleTensor(name: String): OnnxTensor { + val node = style[name] + val values = node["data"].flatMap { batch -> batch.flatMap { row -> row.map { it.floatValue() } } } + return resources.keep(OnnxTensor.createTensor( + environment, FloatBuffer.wrap(values.toFloatArray()), node["dims"].map { it.asLong() }.toLongArray(), + )) + } + val textIds = tensor(arrayOf(ids)) + val textMask = tensor(arrayOf(arrayOf(FloatArray(ids.size) { 1f }))) + val styleDp = styleTensor("style_dp") + val styleTtl = styleTensor("style_ttl") + val duration = sessions[0].run(mapOf( + "text_ids" to textIds, "text_mask" to textMask, "style_dp" to styleDp, + )).use { result -> (result[0] as OnnxTensor).floatBuffer.get(0) / 1.05f } + check(duration.isFinite() && duration > 0 && duration <= 60) { "Invalid speech duration: $duration" } + checkCancelled() + val sampleCount = (duration * sampleRate).toInt() + val latentLength = (sampleCount + chunkSize - 1) / chunkSize + val random = seed?.let(::Random) ?: Random() + var latent = arrayOf(Array(latentDimension) { FloatArray(latentLength) { random.nextGaussian().toFloat() } }) + val latentMask = tensor(arrayOf(arrayOf(FloatArray(latentLength) { 1f }))) + val totalSteps = tensor(floatArrayOf(8f)) + val encoded = resources.keep(sessions[1].run(mapOf( + "text_ids" to textIds, "text_mask" to textMask, "style_ttl" to styleTtl, + ))) + val embedding = encoded[0] as OnnxTensor + repeat(8) { step -> + checkCancelled() + OnnxTensor.createTensor(environment, latent).use { noisy -> + OnnxTensor.createTensor(environment, floatArrayOf(step.toFloat())).use { currentStep -> + sessions[2].run(mapOf( + "noisy_latent" to noisy, "text_emb" to embedding, "style_ttl" to styleTtl, + "latent_mask" to latentMask, "text_mask" to textMask, + "current_step" to currentStep, "total_step" to totalSteps, + )).use { result -> + @Suppress("UNCHECKED_CAST") + latent = result[0].value as Array> + } + } + } + } + checkCancelled() + OnnxTensor.createTensor(environment, latent).use { finalLatent -> + return sessions[3].run(mapOf("latent" to finalLatent)).use { result -> + val samples = (result[0] as OnnxTensor).floatBuffer + check(samples.remaining() >= sampleCount) { "Truncated speech waveform" } + FloatArray(sampleCount).also { samples.get(it) } + } + } + } + } + + override fun close() { + sessions.asReversed().forEach { runCatching { it.close() } } + sessions.clear() + } + + companion object { + const val MAX_CHARS = 300 + val voices = listOf("F1", "F2", "F3", "F4", "F5", "M1", "M2", "M3", "M4", "M5") + + fun modelDirectory(): File { + val override = System.getProperty("yereed.tts.modelDir") ?: System.getenv("YEREED_TTS_MODEL_DIR") + if (!override.isNullOrBlank()) return File(override) + generateSequence(File(System.getProperty("user.dir")).absoluteFile) { it.parentFile }.forEach { root -> + File(root, ".local/tts/supertonic-3").takeIf { it.isDirectory }?.let { return it } + } + return File(System.getProperty("user.home"), ".toread/tts/supertonic-3") + } + + fun isInstalled(directory: File = modelDirectory()): Boolean = + (listOf("onnx/tts.json", "onnx/unicode_indexer.json") + + listOf("duration_predictor", "text_encoder", "vector_estimator", "vocoder").map { "onnx/$it.onnx" } + + voices.map { "voice_styles/$it.json" }).all { File(directory, it).length() > 0 } + + fun languageOf(text: String): String { + val russian = text.any { it in 'А'..'я' || it == 'ё' || it == 'Ё' } + val english = text.any { it in 'a'..'z' || it in 'A'..'Z' } + return when { + russian && english -> "na" + russian -> "ru" + else -> "en" + } + } + + /** Bounded chunks preserve every non-whitespace character, including long unbroken words. */ + fun chunks(text: String): List = buildList { + var remaining = text.trim() + while (remaining.length > MAX_CHARS) { + var end = remaining.lastIndexOfAny(charArrayOf(' ', '\n', '\t'), MAX_CHARS) + if (end <= 0) end = MAX_CHARS + if (Character.isHighSurrogate(remaining[end - 1])) end-- + while (end > 0 && Character.getType(remaining[end]) == Character.NON_SPACING_MARK.toInt()) end-- + require(end > 0) { "Oversized combining-character sequence" } + add(remaining.substring(0, end).trim()) + remaining = remaining.substring(end).trimStart() + } + if (remaining.isNotEmpty()) add(remaining) + } + + internal fun prepareText(text: String, language: String): String { + var normalized = Normalizer.normalize(russianStressToUnicode(text), Normalizer.Form.NFKD) + .replace(Regex("[–‑—]"), "-") + .replace(Regex("[“”]"), "\"") + .replace(Regex("[‘’´`]"), "'") + .replace(Regex("[_\\[\\]|/#→←<>]"), " ") + .replace(Regex("[♥☆♡©\\\\]"), "") + .replace(Regex("[\\x{1F000}-\\x{1FAFF}\\x{2600}-\\x{27BF}]"), "") + if (language == "en") normalized = normalized.replace("@", " at ") + .replace("e.g.,", "for example, ").replace("i.e.,", "that is, ") + normalized = normalized.replace(Regex("\\s+"), " ").trim() + .replace(Regex(" +([,.!?;:'])"), "$1") + if (normalized.isEmpty() || normalized.last() !in ".!?;:,'\")]}…»") normalized += "." + return "<$language>$normalized" + } + + /** RUAccent/Silero notation -> Cyrillic vowel plus U+0301, never a Latin lookalike. */ + internal fun russianStressToUnicode(text: String): String = + text.replace(Regex("\\+([аеёиоуыэюяАЕЁИОУЫЭЮЯ])")) { it.groupValues[1] + "\u0301" } + } +} + +private class NativeResources : AutoCloseable { + private val values = mutableListOf() + fun keep(value: T): T = value.also { values += it } + override fun close() = values.asReversed().forEach { runCatching { it.close() } } +} diff --git a/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/TtsSample.kt b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/TtsSample.kt new file mode 100644 index 0000000..2edbbb1 --- /dev/null +++ b/composeApp/src/jvmMain/kotlin/net/sergeych/toread/tts/TtsSample.kt @@ -0,0 +1,74 @@ +package net.sergeych.toread.tts + +import kotlinx.coroutines.runBlocking +import java.io.File +import java.util.Locale +import kotlin.system.measureNanoTime + +/** Reproducible audition without opening the library, altering books, or contacting a service. */ +fun main(args: Array) = runBlocking { + val output = File(args.firstOrNull() ?: "build/tts-samples") + if ("--silero" in args) { + check(SileroProcess.isInstalled()) { "Install the Silero desktop runtime first." } + SileroProcess().use { engine -> + for ((label, text) in listOf( + "library" to "На двери висел старый замок, а вдали виднелся замок на холме.", + "yo" to "Ежик шел по лесу. Ребенок пьет теплое молоко.", + )) { + val audio = engine.synthesize(text, "xenia") + check(audio.samples.any { kotlin.math.abs(it) > 0.001f }) + val file = File(output, "silero-desktop-$label.wav") + audio.writeWav(file) + println("Silero desktop worker: ${file.absolutePath}; ${audio.samples.size.toDouble() / audio.sampleRate}s") + if ("--play" in args) playDesktopAudio(audio) + } + } + return@runBlocking + } + check(SupertonicTts.isInstalled()) { "Run python3 scripts/download-tts-model.py first." } + lateinit var engine: SupertonicTts + val loadTime = measureNanoTime { engine = SupertonicTts() } / 1e9 + println("Supertonic 3; CPU only; up to 4 threads; 8 steps; model load: %.3f s".format(Locale.ROOT, loadTime)) + val stressTrial = "--stress" in args + val ruaccentTrial = "--ruaccent" in args + val russianText = "В библиотеке было тихо. Она открыла книгу и начала читать. За окном шёл осенний дождь. На двери висел старый замок, а вдали виднелся замок на холме. Завтра, подумала она, мы наконец доберёмся до моря." + val accentor = if (ruaccentTrial) RuAccentProcess() else null + val samples = if (ruaccentTrial) linkedMapOf("ru-plain" to russianText, "ru-ruaccent" to russianText) else if (stressTrial) linkedMapOf( + "ru-plain" to "Перед нами старый замок. Они стояли около замка.", + "ru-castle" to "П+еред н+ами ст+арый з+амок. Он+и сто+яли +около з+амка.", + "ru-lock" to "П+еред н+ами ст+арый зам+ок. Он+и сто+яли +около замк+а.", + ) else linkedMapOf( + "en" to "The library was quiet. She opened the book and began to read. Outside, the autumn rain tapped softly against the window. Tomorrow, she thought, they would finally reach the coast.", + "ru" to russianText, + ) + engine.use { + try { + for (voice in if (stressTrial || ruaccentTrial) listOf("F1") else listOf("F1", "M1")) for ((label, text) in samples) { + val language = if (stressTrial || ruaccentTrial) "ru" else label + val accentStart = System.nanoTime() + val accented = if (label == "ru-ruaccent") accentor!!.accentuate(text) else text + if (label == "ru-ruaccent") println("RuAccent (including first load): %.3f s".format(Locale.ROOT, (System.nanoTime() - accentStart) / 1e9)) + val spoken = SupertonicTts.russianStressToUnicode(accented) + lateinit var audio: SpeechAudio + val elapsed = measureNanoTime { + audio = joinSpeechAudio(SupertonicTts.chunks(spoken).map { chunk -> + SpeechAudio(engine.synthesize(chunk, voice, language, seed = 42), engine.sampleRate) + }) + } / 1e9 + val duration = audio.samples.size.toDouble() / audio.sampleRate + check(audio.samples.all { it.isFinite() } && audio.samples.any { kotlin.math.abs(it) > 0.001f }) { + "Invalid or silent model output" + } + val file = File(output, "$label-$voice.wav") + audio.writeWav(file) + File(output, "$label-$voice.txt").writeText(spoken + "\n") + println("$label/$voice: %.3f s synthesis, %.3f s audio, RTF %.3f; %s".format( + Locale.ROOT, elapsed, duration, elapsed / duration, file.absolutePath, + )) + if ("--play" in args) playDesktopAudio(audio) + } + } finally { + accentor?.close() + } + } +} diff --git a/composeApp/src/jvmMain/resources/META-INF/licenses/supertonic-code-MIT.txt b/composeApp/src/jvmMain/resources/META-INF/licenses/supertonic-code-MIT.txt new file mode 100644 index 0000000..943171b --- /dev/null +++ b/composeApp/src/jvmMain/resources/META-INF/licenses/supertonic-code-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Supertone Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/composeApp/src/jvmMain/resources/tts/ruaccent-worker.py b/composeApp/src/jvmMain/resources/tts/ruaccent-worker.py new file mode 100644 index 0000000..daaf700 --- /dev/null +++ b/composeApp/src/jvmMain/resources/tts/ruaccent-worker.py @@ -0,0 +1,56 @@ +"""Local stdin/stdout JSON protocol. Book text never leaves this process tree.""" +import contextlib +import json +import os +from pathlib import Path +import sys +import time + +os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_TELEMETRY="1", TOKENIZERS_PARALLELISM="false") + + +def load(model): + # Bound ORT worker counts before RUAccent imports its session constructor. + import onnxruntime as ort + original = ort.InferenceSession + + def cpu_session(path, *args, **kwargs): + options = ort.SessionOptions() + options.intra_op_num_threads = 4 + options.inter_op_num_threads = 1 + kwargs["sess_options"] = options + kwargs["providers"] = ["CPUExecutionProvider"] + return original(path, *args, **kwargs) + + ort.InferenceSession = cpu_session + from ruaccent import RUAccent + accentor = RUAccent() + accentor.load(omograph_model_size="turbo3.1", use_dictionary=True, device="CPU", workdir=str(model), tiny_mode=False) + return accentor + + +def main(): + with contextlib.redirect_stdout(sys.stderr): + accentor = load(Path(sys.argv[1])) + if "--check" in sys.argv: + with contextlib.redirect_stdout(sys.stderr): + result = accentor.process_all("На двери висит замок.") + print(result, flush=True) + return + print(json.dumps({"ready": True}), flush=True) + for line in sys.stdin: + try: + text = json.loads(line)["text"] + if not isinstance(text, str) or len(text) > 2000: + raise ValueError("Expected text of at most 2000 characters") + start = time.monotonic() + with contextlib.redirect_stdout(sys.stderr): + accented = accentor.process_all(text) + print(json.dumps({"text": accented, "seconds": time.monotonic() - start}, ensure_ascii=False), flush=True) + except Exception as error: + # Do not echo private input or exception text into application logs. + print(json.dumps({"error": type(error).__name__}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/composeApp/src/jvmMain/resources/tts/silero-worker.py b/composeApp/src/jvmMain/resources/tts/silero-worker.py new file mode 100644 index 0000000..3f477c1 --- /dev/null +++ b/composeApp/src/jvmMain/resources/tts/silero-worker.py @@ -0,0 +1,47 @@ +"""Persistent CPU-only Silero worker; local JSON lines, float32 audio in base64.""" +import array +import base64 +import contextlib +import json +import os +from pathlib import Path +import re +import sys + +os.environ["CUDA_VISIBLE_DEVICES"] = "" +os.environ["TORCH_HOME"] = str(Path(sys.argv[1]).parent / "torch") + + +def main(): + with contextlib.redirect_stdout(sys.stderr): + import torch + torch.set_num_threads(4) + torch.set_num_interop_threads(1) + model = torch.package.PackageImporter(str(Path(sys.argv[1]) / "model.pt")).load_pickle("tts_models", "model") + model.to(torch.device("cpu")) + print(json.dumps({"ready": True}), flush=True) + for line in sys.stdin: + try: + request = json.loads(line) + text = request["text"] + voice = request["voice"] + if not isinstance(text, str) or not 0 < len(text) <= 2000 or voice not in model.speakers: + raise ValueError("Invalid speech request") + # Preserve explicit reader/book stress in Silero's native notation. + text = re.sub(r"([аеёиоуыэюяАЕЁИОУЫЭЮЯ])\u0301", r"+\1", text) + with contextlib.redirect_stdout(sys.stderr), torch.inference_mode(): + audio = model.apply_tts(text=text, speaker=voice, sample_rate=48000, + put_accent=True, put_yo=True, + put_stress_homo=True, put_yo_homo=True).detach().cpu().flatten() + if not audio.numel() or not torch.isfinite(audio).all(): + raise ValueError("Invalid model audio") + samples = array.array("f", audio.tolist()) + if sys.byteorder != "little": + samples.byteswap() + print(json.dumps({"sampleRate": 48000, "audio": base64.b64encode(samples.tobytes()).decode("ascii")}), flush=True) + except Exception as error: + print(json.dumps({"error": type(error).__name__}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayerTest.kt b/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayerTest.kt new file mode 100644 index 0000000..8f3fec7 --- /dev/null +++ b/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/DesktopReadAloudPlayerTest.kt @@ -0,0 +1,186 @@ +package net.sergeych.toread.tts + +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import net.sergeych.toread.ReadAloudSentence +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +@OptIn(ExperimentalCoroutinesApi::class) +class DesktopReadAloudPlayerTest { + private fun sentence(index: Int, text: String) = ReadAloudSentence(index, index, 0, text.length, text) + private fun audio(id: Int) = SpeechAudio(floatArrayOf(id.toFloat()), 24000) + + @Test + fun sileroDefaultsToXeniaRoutesEnglishAndResetsVoiceOnEngineChange() = runTest { + val calls = mutableListOf() + var accented = false + val player = DesktopReadAloudPlayer(this, { text, voice -> calls += "super:$voice:$text"; audio(1) }, + {}, { true }, { accented = true; it }, { text, voice -> calls += "silero:$voice:$text"; audio(1) }) + assertEquals("silero", player.settingsState.value.selectedEngineId) + assertEquals("xenia", player.settingsState.value.selectedVoiceId) + player.prepare(listOf(sentence(0, "Привет."), sentence(1, "Hello."), sentence(2, "Привет, John.")), 0) + player.play() + advanceUntilIdle() + assertEquals(listOf("silero:xenia:Привет.", "super:F1:Hello.", "super:F1:Привет, John."), calls) + assertFalse(accented) + player.selectVoice("F1") + assertEquals("xenia", player.settingsState.value.selectedVoiceId) + player.selectEngine("supertonic-3") + assertEquals("F1", player.settingsState.value.selectedVoiceId) + player.selectEngine("silero") + assertEquals("xenia", player.settingsState.value.selectedVoiceId) + } + + @Test + fun switchingAwayFromSileroCancelsPendingSpeech() = runTest { + var cancelled = false + var played = 0 + val player = DesktopReadAloudPlayer(this, { _, _ -> audio(1) }, { played++ }, { true }, + silero = { _, _ -> try { awaitCancellation() } finally { cancelled = true } }) + player.prepare(listOf(sentence(0, "Привет.")), 0) + player.play() + runCurrent() + player.selectEngine("supertonic-3") + advanceUntilIdle() + assertTrue(cancelled) + assertEquals(1, played) + } + + @Test + fun stopDuringSynthesisDiscardsAudioAndDoesNotReportAnError() = runTest { + var started = false + var played = false + val player = DesktopReadAloudPlayer(this, { _, _ -> started = true; awaitCancellation() }, { played = true }, { true }) + player.prepare(listOf(sentence(7, "First.")), 0) + player.play() + runCurrent() + assertTrue(started) + assertTrue(player.state.value.buffering) + player.stop() + runCurrent() + assertFalse(played) + assertFalse(player.state.value.active) + assertNull(player.settingsState.value.message) + } + + @Test + fun skipCancelsOldSynthesisAndStartsRequestedSentence() = runTest { + val played = mutableListOf() + val player = DesktopReadAloudPlayer(this, { text, _ -> + if (text == "First.") awaitCancellation() else audio(2) + }, { played += it.samples[0].toInt(); awaitCancellation() }, { true }) + player.prepare(listOf(sentence(10, "First."), sentence(20, "Second.")), 0) + player.play() + runCurrent() + player.skip(1) + runCurrent() + assertEquals(listOf(2), played) + assertEquals(20, player.state.value.sentenceIndex) + player.stop() + runCurrent() + } + + @Test + fun voiceChangeCancelsPlaybackAndRestartsCurrentSentence() = runTest { + val voices = mutableListOf() + var cancelledAudio = 0 + val player = DesktopReadAloudPlayer(this, { _, voice -> voices += voice; audio(1) }, { + try { awaitCancellation() } finally { cancelledAudio++ } + }, { true }) + player.prepare(listOf(sentence(3, "Hello.")), 0) + player.play() + runCurrent() + player.selectVoice("M1") + runCurrent() + assertEquals(listOf("F1", "M1"), voices) + assertEquals(1, cancelledAudio) + assertEquals(3, player.state.value.sentenceIndex) + player.stop() + runCurrent() + assertEquals(2, cancelledAudio) + } + + @Test + fun lookaheadDoesNotAdvanceHighlightBeforePlayback() = runTest { + val synthesized = mutableListOf() + val player = DesktopReadAloudPlayer(this, { text, _ -> synthesized += text; audio(1) }, { + awaitCancellation() + }, { true }) + player.prepare(listOf(sentence(4, "First."), sentence(8, "Second."), sentence(12, "Third.")), 0) + player.play() + runCurrent() + assertEquals(listOf("First.", "Second."), synthesized) + assertEquals(4, player.state.value.sentenceIndex) + player.stop() + runCurrent() + } + + @Test + fun fullQueueUsesSpokenTextAndFinishes() = runTest { + val synthesized = mutableListOf() + val player = DesktopReadAloudPlayer(this, { text, _ -> synthesized += text; audio(1) }, { delay(10) }, { true }) + player.prepare(listOf(sentence(0, "1").copy(spokenText = "One.", pauseBeforeMillis = 20), sentence(1, "Two.")), 0) + player.play() + advanceUntilIdle() + assertEquals(listOf("One.", "Two."), synthesized) + assertFalse(player.state.value.playing) + assertFalse(player.state.value.active) + } + + @Test + fun accentStageCanBeComparedWithPlainEngineAndNeverChangesOriginalText() = runTest { + val inputs = mutableListOf() + val original = sentence(0, "замок") + val player = DesktopReadAloudPlayer(this, { text, _ -> inputs += text; audio(1) }, {}, { true }, { "з+амок" }) + player.prepare(listOf(original), 0) + player.play() + advanceUntilIdle() + assertEquals(listOf("за\u0301мок"), inputs) + assertEquals("замок", original.spokenText) + player.selectEngine("supertonic-3") + player.play() + advanceUntilIdle() + assertEquals(listOf("за\u0301мок", "замок"), inputs) + } + + @Test + fun accentedTextIsRechunkedToFitTtsWithoutDiscardingMarks() = runTest { + val inputs = mutableListOf() + val marked = "а\u0301 ".repeat(110).trim() + val player = DesktopReadAloudPlayer(this, { text, _ -> inputs += text; audio(1) }, {}, { true }, { marked }) + player.prepare(listOf(sentence(0, "а ".repeat(110))), 0) + player.play() + advanceUntilIdle() + assertTrue(inputs.size > 1) + assertTrue(inputs.all { it.length <= 300 }) + assertEquals(marked.replace(" ", ""), inputs.joinToString("").replace(" ", "")) + } + + @Test + fun missingModelAndInferenceErrorsRemainVisibleAndRetryable() = runTest { + var installed = false + var calls = 0 + val player = DesktopReadAloudPlayer(this, { _, _ -> calls++; error("bad model") }, {}, { installed }) + player.prepare(listOf(sentence(0, "Hello.")), 0) + player.play() + assertEquals(0, calls) + assertFalse(player.state.value.playing) + assertNotNull(player.settingsState.value.message) + installed = true + player.play() + advanceUntilIdle() + assertEquals(1, calls) + assertTrue(player.state.value.active) + assertFalse(player.state.value.buffering) + assertNotNull(player.settingsState.value.message) + } +} diff --git a/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/SupertonicTextTest.kt b/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/SupertonicTextTest.kt new file mode 100644 index 0000000..e3ee93a --- /dev/null +++ b/composeApp/src/jvmTest/kotlin/net/sergeych/toread/tts/SupertonicTextTest.kt @@ -0,0 +1,45 @@ +package net.sergeych.toread.tts + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SupertonicTextTest { + @Test + fun longSentencesAndUnbrokenWordsAreBoundedWithoutLosingText() { + for (text in listOf("Слово ".repeat(150), "a".repeat(601), "a".repeat(299) + "😀" + "b".repeat(301), "а".repeat(300) + "\u0301" + "а".repeat(50))) { + val chunks = SupertonicTts.chunks(text) + assertTrue(chunks.all { it.length <= 300 && it.isNotBlank() }) + assertEquals(text.filterNot(Char::isWhitespace), chunks.joinToString("").filterNot(Char::isWhitespace)) + assertTrue(chunks.none { Character.isHighSurrogate(it.last()) || Character.isLowSurrogate(it.first()) }) + assertTrue(chunks.none { it.first() == '\u0301' }) + } + } + + @Test + fun languageSelectionHandlesRussianEnglishAndMixedText() { + assertEquals("ru", SupertonicTts.languageOf("Ёж шёл домой.")) + assertEquals("en", SupertonicTts.languageOf("The library.")) + assertEquals("na", SupertonicTts.languageOf("Он сказал Hello.")) + } + + @Test + fun preprocessingDoesNotInsertPredictedStress() { + assertEquals("На двери висит замок.", SupertonicTts.prepareText("На двери висит замок", "ru")) + assertEquals("Hello!", SupertonicTts.prepareText("Hello !", "en")) + } + + @Test + fun ruaccentStressBecomesCombiningAcuteOnCyrillicVowels() { + assertEquals("за\u0301мок замо\u0301к Ё\u0301лка", SupertonicTts.russianStressToUnicode("з+амок зам+ок +Ёлка")) + assertEquals("2+2 +a +б а\u0301", SupertonicTts.russianStressToUnicode("2+2 +a +б а\u0301")) + assertEquals("За\u0301мок.", SupertonicTts.prepareText("З+амок", "ru")) + assertEquals("За\u0301мок.", SupertonicTts.prepareText("За\u0301мок", "ru")) + } + + @Test + fun pcmConversionClipsAndHandlesNonFiniteSamples() { + val bytes = SpeechAudio(floatArrayOf(-2f, 2f, Float.NaN), 44100).pcm16() + assertEquals(listOf(1, 128, 255, 127, 0, 0), bytes.map { it.toInt() and 255 }) + } +} diff --git a/docs/desktop-tts.md b/docs/desktop-tts.md new file mode 100644 index 0000000..118bde6 --- /dev/null +++ b/docs/desktop-tts.md @@ -0,0 +1,176 @@ +# Desktop CPU speech trial + +Desktop Read aloud defaults to **Silero v5_5_ru / Xenia** for Russian when its +local runtime is installed. Silero runs in a persistent CPU-only Python process +and handles its own stress and ё restoration. English and mixed-script chunks +use Supertonic 3 / F1 through ONNX Runtime's CPU provider. Mixed passages retain +Supertonic's Russian pronunciation limitations. Plain Supertonic is retained as an audition baseline. Initial listening feedback +found its Russian word stress unacceptable. An optional RUAccent CPU stage now +adds stress and restores ё before synthesis; its +vowel notation is converted to +a Cyrillic vowel followed by U+0301. Manual minimal-pair listening showed stress +changes, but this is not a comprehensive pronunciation-quality evaluation. +Android retains its existing system TTS implementation. + +## Run + +From the repository root: + +```sh +python3 scripts/download-tts-model.py +./gradlew :composeApp:run +``` + +The script downloads about 401 MB into the ignored `.local/tts/supertonic-3` +directory. Downloads are revision-pinned and verified against the committed file +manifest. It reuses valid files and replaces incomplete downloads only after hash +verification. No Python packages are needed. The model license is saved alongside +its weights. + +Open a book and choose **Read aloud** from the reader menu. Playback highlights the +current sentence, follows the reading position, and exposes Stop, previous/next +sentence, and voice settings. Silero offers Xenia (default), Baya, Aidar, Kseniya +and Eugene. The Supertonic engines offer F1–F5 and M1–M5. Engine and voice choices +last for this application session; each launch starts with Xenia if installed. + +The model loads lazily on first playback. One chunk is prepared ahead while audio +plays. Stop, skipping, and voice changes cancel the old playback and lookahead; +Supertonic inference checks cancellation between model stages/denoising steps; +cancelling Silero inference terminates its worker, which restarts on demand. Long +sentences are split at whitespace into at most 300-character chunks. English and +Russian are selected from their scripts; mixed-script chunks use the model's +language-agnostic `na` mode. This heuristic is not general language detection. + +For a model installed elsewhere, set `YEREED_TTS_MODEL_DIR` (or JVM property +`yereed.tts.modelDir`) to the directory containing `onnx/` and `voice_styles/`. +Without an override the app searches parent directories for the workspace model, +then falls back to `~/.toread/tts/supertonic-3`. Packaged applications can use that +home-directory location. The application itself never downloads models or sends +text over the network. The first Gradle build and model setup require downloads. + +## Optional RUAccent stage (desktop trial) + +```sh +python3 scripts/setup-ruaccent.py +./gradlew :composeApp:run +``` + +This creates a private Python 3.12 runtime under `.local/ruaccent-runtime` and +fetches about 726 MB of revision-pinned RUAccent assets into `.local/tts/ruaccent`. +It does not change system Python. The bootstrap supports Linux x86_64 directly; +on other systems install `uv` first. The dependency versions and model hashes +are recorded under `scripts/`. This desktop trial uses RUAccent 1.5.8.3, +`turbo3.1`, the full dictionary/rule pipeline, and CPU execution with four threads. +This is a desktop quality experiment, not an Android footprint recommendation. + +After setup, restart the reader. **Supertonic 3 + RUAccent** becomes the default +engine; select **Supertonic 3** in read-aloud settings for a direct comparison. +English bypasses the accentor. Already marked Russian chunks are preserved. +The accentor is a persistent local child process communicating over stdin/stdout; +Hugging Face and Transformers run in offline mode. Book contents are never sent +to a server. Set `YEREED_RUACCENT_HOME` to override its installation directory. +Stop/skip kills a pending accentor request when necessary; the next request +restarts the worker. Displayed book text and stored files are unchanged. + +The model recognizes U+0301, and preprocessing preserves it. For example, +`з+амок` becomes `за́мок`, not a Latin accented-vowel substitution. Predicted +stress can still be wrong; model-token support does not guarantee compliance. + +## Audition / benchmark + +```sh +./gradlew :composeApp:ttsSample +# Also play the four samples through the desktop audio device: +./gradlew :composeApp:ttsSample -PttsPlay=true +# Controlled manual stress pairs, same words/noise seed: +./gradlew :composeApp:ttsSample -PttsStress=true -PttsPlay=true +# Original plain Russian passage, then RUAccent + Supertonic: +./gradlew :composeApp:ttsSample -PttsRuAccent=true -PttsPlay=true +``` + +WAV files and their exact unaccented texts go to `composeApp/build/tts-samples/`. +The sample task uses the same inference and audio code as the reader without +opening or changing the library. It prints model-load time, synthesis time, +audio duration, and real-time factor (generation seconds / audio seconds). + +Initial measurements on an Intel Core Ultra 9 275HX, four CPU threads, eight +steps, speed 1.05: 1.075 seconds model load; English/Russian F1/M1 samples generated +11.6–15.3 seconds of audio in 2.4–3.3 seconds (RTF 0.20–0.24). These are short +local samples, not phone/battery or long-book benchmarks. Synthesis completion +and non-silent waveforms do not establish pronunciation quality. + +## Dependencies and provenance + +- [Supertonic source](https://github.com/supertone-oss-archive/supertonic): the + Kotlin inference code adapts the MIT Java example; attribution is in + [licenses/supertonic-code-MIT.txt](licenses/supertonic-code-MIT.txt). +- [Supertonic 3 model](https://huggingface.co/supertone-oss-archive/supertonic-3): + revision `aafc6e32416a594460b32413efc49d7fe4ce6d46`, OpenRAIL-M weights. + Upstream development/support ended in September 2026. +- ONNX Runtime 1.23.1, standard CPU artifact; no CUDA/GPU provider. +- JSON loading uses Jackson; PCM playback uses Java Sound. + +Run `./gradlew :composeApp:jvmTest` for text-boundary, PCM conversion, queue, +lookahead, cancellation, voice-change, and missing-model/error tests, together +with existing desktop/common tests. Model inference is exercised separately by +`ttsSample`, so ordinary tests do not download weights or use speakers. + +The isolated virtual-display capture did not provide a usable UI image; rendered +GUI verification remains incomplete. No further screen access is used. + +RUAccent measurements on the same host: about 5.9 seconds cold load, 30–43 ms +for the two warm short passages, and about 2.1 GiB resident memory for the full +accentor worker alone. The combined passage took 2.98 seconds of TTS synthesis +for 16.42 seconds of audio, excluding accentor startup. In that passage RUAccent +incorrectly chose `за́мок` for the door lock; on a shorter explicit door-lock +sentence it chose `замо́к`. These are limited observations, not an accuracy score. + +## Silero Russian audition + +User listening feedback rejected Supertonic's Russian quality. A separate CPU +audition uses official Silero `v5_5_ru`, with its own stress, ё, stress-homograph +and ё-homograph processing enabled. It does not use RUAccent. The same model is +now the preferred desktop Russian engine. The Russian model is CC BY-NC-SA 4.0, unlike Silero's MIT CIS +base models; see [upstream](https://github.com/snakers4/silero-models). + +Setup from the repository root (requires `uv`; our existing local copy is +`.local/tools/uv`): + +```sh +export UV_CACHE_DIR="$PWD/.local/uv-cache" +export UV_PYTHON_INSTALL_DIR="$PWD/.local/python" +export UV_SYSTEM_CERTS=true +.local/tools/uv venv --python 3.12.14 .local/silero-runtime +.local/tools/uv pip sync --python .local/silero-runtime/bin/python scripts/silero-requirements.txt --index-url https://download.pytorch.org/whl/cpu +mkdir -p .local/tts/silero-v5_5-ru +curl -fL https://models.silero.ai/models/tts/ru/v5_5_ru.pt -o .local/tts/silero-v5_5-ru/model.pt.part +mv .local/tts/silero-v5_5-ru/model.pt.part .local/tts/silero-v5_5-ru/model.pt +.local/silero-runtime/bin/python scripts/silero-sample.py +python3 - <<'PY' +from pathlib import Path +import json +root = Path.cwd() +(root / '.local/tts/silero-v5_5-ru/runtime.json').write_text(json.dumps({ + 'python': str(root / '.local/silero-runtime/bin/python') +}) + '\n') +PY +# Restart the reader after setup, or check its actual Kotlin/Python bridge: +./gradlew :composeApp:ttsSample -PttsSilero=true +``` + +Outputs: `composeApp/build/tts-samples/silero-*.wav`, matching input `.txt` +files, and `silero-benchmark.json` with timings, settings and the downloaded +model's SHA-256. The script uses CPU-only PyTorch, four threads and 48 kHz PCM. +Default voices are Xenia, Baya and Aidar; `--voices xenia` limits the audition. +Each reads the earlier library passage and a separate unaccented ё/context +challenge. Pronunciation quality must be judged by listening. + +The desktop integration discovers `silero-v5_5-ru` beside the Supertonic model +directory; `YEREED_SILERO_HOME` can override it. `runtime.json` points to the +isolated Python executable. The worker script is bundled in the application; +model loading and inference are local. Both models are needed for bilingual use. + +Initial audition on the same desktop: model load 0.38 seconds; six samples +generated 13.2–15.5 seconds of audio in 0.36–0.82 seconds (RTF 0.027–0.059, +including the first synthesis). All outputs were finite and non-silent. +Downloaded model SHA-256: +`50081637b602126ee06cb3bc8a744d25651d2da149ee8864b9a379bfdd934437`. diff --git a/docs/licenses/supertonic-code-MIT.txt b/docs/licenses/supertonic-code-MIT.txt new file mode 100644 index 0000000..943171b --- /dev/null +++ b/docs/licenses/supertonic-code-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Supertone Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 472124e..87ffcca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,8 +19,12 @@ ktor = "3.4.3" logback = "1.5.32" material3 = "1.11.0-alpha07" h2 = "2.4.240" +onnxruntime = "1.23.1" +jackson = "2.18.6" [libraries] +onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnxruntime" } +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } kotlin-testJunit = { module = "org.jetbrains.kotlin:kotlin-test-junit", version.ref = "kotlin" } junit = { module = "junit:junit", version.ref = "junit" } @@ -41,6 +45,7 @@ compose-components-resources = { module = "org.jetbrains.compose.components:comp compose-uiToolingPreview = { module = "org.jetbrains.compose.ui:ui-tooling-preview", version.ref = "composeMultiplatform" } kotlinx-coroutinesSwing = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-coroutinesCore = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } +kotlinx-coroutinesTest = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } h2 = { module = "com.h2database:h2", version.ref = "h2" } logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } ktor-serverCore = { module = "io.ktor:ktor-server-core-jvm", version.ref = "ktor" } diff --git a/scripts/download-tts-model.py b/scripts/download-tts-model.py new file mode 100644 index 0000000..309f766 --- /dev/null +++ b/scripts/download-tts-model.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Download the pinned Supertonic 3 CPU model. No packages or credentials required.""" +import hashlib +import json +from pathlib import Path +import sys +import urllib.request + +ROOT = Path(__file__).resolve().parent.parent +MANIFEST = json.loads((ROOT / "scripts/supertonic-3-model.json").read_text()) + + +def valid(path, entry): + if not path.is_file() or path.stat().st_size != entry["size"]: + return False + digest = hashlib.sha256() if "sha256" in entry else hashlib.sha1() + if "git_blob" in entry: + digest.update(f"blob {entry['size']}\0".encode()) + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == entry.get("sha256", entry.get("git_blob")) + + +def download(manifest, destination): + total = sum(entry["size"] for entry in manifest["files"]) + print(f"{manifest['repository']}: {total / 1e6:.1f} MB; destination: {destination}", flush=True) + base = f"https://huggingface.co/{manifest['repository']}/resolve/{manifest['revision']}" + for entry in manifest["files"]: + target = destination / entry["path"] + if valid(target, entry): + print(f"Verified {entry['path']}", flush=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(target.suffix + ".part") + print(f"Downloading {entry['path']} ({entry['size'] / 1e6:.1f} MB)", flush=True) + try: + with urllib.request.urlopen(f"{base}/{entry['path']}", timeout=60) as source, temporary.open("wb") as output: + while chunk := source.read(1024 * 1024): + output.write(chunk) + if not valid(temporary, entry): + raise RuntimeError(f"Checksum mismatch: {entry['path']}") + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + + +def main(): + destination = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / ".local/tts/supertonic-3" + print("Supertonic model license: OpenRAIL-M (saved as LICENSE alongside the model).", flush=True) + download(MANIFEST, destination) + print("Ready. Run ./gradlew :composeApp:run and use Read aloud in a book.", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/ruaccent-model.json b/scripts/ruaccent-model.json new file mode 100644 index 0000000..87eaae7 --- /dev/null +++ b/scripts/ruaccent-model.json @@ -0,0 +1,231 @@ +{ + "repository": "ruaccent/accentuator", + "revision": "b78ae5ea1e62beaf138bed1865cd8c3b0b5ca855", + "files": [ + { + "path": "README.md", + "size": 2660, + "git_blob": "7f12645cf20ed21742fd63a3b799efd8c7007734" + }, + { + "path": "dictionary/accents.json.gz", + "size": 20954156, + "sha256": "aa460ebba90de00fbbf3d41d121961f605b98667e45efb7920f127473b15515e" + }, + { + "path": "dictionary/accents_nn.json.gz", + "size": 845996, + "sha256": "8395664000b80c1afe09bfea3650945b0933482b8e3dee5bb9d429eb18c44935" + }, + { + "path": "dictionary/omographs.json.gz", + "size": 219047, + "sha256": "04a9e81c68d65f65ba493fe0110f99e79087548c2beeec3032e2b66e28706f36" + }, + { + "path": "dictionary/rule_engine/accents.json", + "size": 694782, + "git_blob": "91690536e14e133fe4017941a466fde663b96011" + }, + { + "path": "dictionary/rule_engine/forms.json", + "size": 2669, + "git_blob": "6686cd615fb55c09976550aeba3ad6687ecc32dd" + }, + { + "path": "dictionary/yo_homographs.json.gz", + "size": 5747, + "sha256": "c4ee777bbbab87f9eac838f370ad92974e079d02b21903e480c54b5f0c8c60d1" + }, + { + "path": "dictionary/yo_words.json.gz", + "size": 548914, + "sha256": "a19fa89a964a0691d9fe4ee384783e3934904891843d8f59a1c480d67947a82a" + }, + { + "path": "koziev/rulemma/rulemma.dat", + "size": 16703198, + "sha256": "bf2b3ef3ff7a0aa6e4250aa4e9c8ed568e25f825deebdb12dee1b46b785ba9fc" + }, + { + "path": "koziev/rulemma/rulemma.py", + "size": 11166, + "git_blob": "a182b7d1694b13e8ac6807e2701ad17bf4064bbe" + }, + { + "path": "koziev/rupostagger/__init__.py", + "size": 111, + "git_blob": "01bd02ccfc68c851542390a6d0a684481f897bb9" + }, + { + "path": "koziev/rupostagger/database/ruword2tags.db", + "size": 168816640, + "sha256": "a06848e656bef642aafb4440c03554fa78f2f32dde92ea66f3f86ce9977b167e" + }, + { + "path": "koziev/rupostagger/rupostagger.config", + "size": 276, + "git_blob": "4794511fc1b5ef4bc649fab5eafdf6b775206b72" + }, + { + "path": "koziev/rupostagger/rupostagger.model", + "size": 2417464, + "sha256": "21b7b0bfd7427b5fdc1604052176db8aa3b139b3ce03be440cfce48536f8e5ef" + }, + { + "path": "koziev/rupostagger/rupostagger.py", + "size": 7118, + "git_blob": "198a8db45678a9578dfad5bbf01d3efd8e95a7c8" + }, + { + "path": "koziev/rupostagger/rusyllab.py", + "size": 18975, + "git_blob": "7f606a48ff2a5de6b6388a45caf1d41bd6089611" + }, + { + "path": "koziev/rupostagger/ruword2tags.dat", + "size": 9683765, + "sha256": "dde47b5f1d48ff899887ac07812dcabd2966e48e84646f3065bfd06627c2af58" + }, + { + "path": "koziev/rupostagger/ruword2tags.py", + "size": 18086, + "git_blob": "71098637747f19e44788506775e4cd1e6c5e7183" + }, + { + "path": "nn/nn_accent/big.onnx", + "size": 2285217, + "sha256": "47e69d9ae19f2a82e21b1c70f6a4bbfb1abc5759e98b2e67d009c5e9d7af18c9" + }, + { + "path": "nn/nn_accent/config.json", + "size": 841, + "git_blob": "0d1656a1fb93fcea726186c8bfad977c6e27c4a5" + }, + { + "path": "nn/nn_accent/model.onnx", + "size": 803402, + "sha256": "4e393144e45626f6f1062a0784ef06f921b97321a8e7b87ac2a09a892286500a" + }, + { + "path": "nn/nn_accent/ort_config.json", + "size": 727, + "git_blob": "5a8fc726323dda820ae4e794e4d0d514de07eee2" + }, + { + "path": "nn/nn_accent/special_tokens_map.json", + "size": 99, + "git_blob": "1543e951b7c6f9c246a5e441d7162b2c65ab48a5" + }, + { + "path": "nn/nn_accent/tokenizer_config.json", + "size": 257, + "git_blob": "5ea10565e634adcd37089652fbf6bb2cf05c1530" + }, + { + "path": "nn/nn_accent/vocab.txt", + "size": 140, + "git_blob": "1e5a3d765a6f5a686ed3c6294efb85e9e00a431a" + }, + { + "path": "nn/nn_omograph/turbo3.1/added_tokens.json", + "size": 279418, + "git_blob": "23620bf92f2ad9e1243202cff127617fe9415c9a" + }, + { + "path": "nn/nn_omograph/turbo3.1/config.json", + "size": 723, + "git_blob": "2ff7dbeb01fffcb05f65ad430b08205fe6668693" + }, + { + "path": "nn/nn_omograph/turbo3.1/merges.txt", + "size": 1213606, + "git_blob": "2d4658a914a3760fbf1a3a9c72b42eefcf8d8ded" + }, + { + "path": "nn/nn_omograph/turbo3.1/model.onnx", + "size": 359306923, + "sha256": "2cb6a174c4cdb45bd3132b4f7c8a3779fc4b6869863180ed7d0e421bcd453dbd" + }, + { + "path": "nn/nn_omograph/turbo3.1/special_tokens_map.json", + "size": 280, + "git_blob": "d5698132694f4f1bcff08fa7d937b1701812598e" + }, + { + "path": "nn/nn_omograph/turbo3.1/tokenizer.json", + "size": 5532343, + "git_blob": "31ac709e3c4625c6bf80b7c97066a9cde4d9ba89" + }, + { + "path": "nn/nn_omograph/turbo3.1/tokenizer_config.json", + "size": 492, + "git_blob": "15a19568f93d1b292c7ff6d4ef0fdac13bd65b53" + }, + { + "path": "nn/nn_omograph/turbo3.1/vocab.json", + "size": 1555282, + "git_blob": "ede65202ec9ef483ff166bb31f76615f30048db7" + }, + { + "path": "nn/nn_stress_usage_predictor/config.json", + "size": 822, + "git_blob": "df189246d87953bead249c2ee82e3985a96be8f8" + }, + { + "path": "nn/nn_stress_usage_predictor/model.onnx", + "size": 116473561, + "sha256": "3d547500637b4ddfec8880ed6d1405fd50ee9d3f0131ef8a2a69dcf961dbefeb" + }, + { + "path": "nn/nn_stress_usage_predictor/special_tokens_map.json", + "size": 125, + "git_blob": "a8b3208c2884c4efb86e49300fdd3dc877220cdf" + }, + { + "path": "nn/nn_stress_usage_predictor/tokenizer.json", + "size": 2413536, + "git_blob": "b4d5ccba4a193e0330537378bb0bba47026bd364" + }, + { + "path": "nn/nn_stress_usage_predictor/tokenizer_config.json", + "size": 368, + "git_blob": "d7d40c5e58b2c6ba58974571f617abe23221eb6c" + }, + { + "path": "nn/nn_stress_usage_predictor/vocab.txt", + "size": 1080667, + "git_blob": "211012a925fc51581997b6d3ef047bcda9a842f4" + }, + { + "path": "nn/nn_yo_homograph_resolver/config.json", + "size": 625, + "git_blob": "6615961d500dfe1fbbf302d1bbd4832b7e27f398" + }, + { + "path": "nn/nn_yo_homograph_resolver/model.onnx", + "size": 14332169, + "sha256": "42cc85bf0c4b319dfe3d89fa17b162a92fd5e1c651a657cb3d5f44978d4e70ac" + }, + { + "path": "nn/nn_yo_homograph_resolver/special_tokens_map.json", + "size": 125, + "git_blob": "a8b3208c2884c4efb86e49300fdd3dc877220cdf" + }, + { + "path": "nn/nn_yo_homograph_resolver/tokenizer.json", + "size": 126952, + "git_blob": "bbab1a4bc4dfa68697338718621489e77d03e70a" + }, + { + "path": "nn/nn_yo_homograph_resolver/tokenizer_config.json", + "size": 401, + "git_blob": "7d8a1ab4371909b9dcaabfcf93b9bbd56045605a" + }, + { + "path": "nn/nn_yo_homograph_resolver/vocab.txt", + "size": 48576, + "git_blob": "32bc54d43a46178e70dcefb85626f52e9ddbc98b" + } + ] +} diff --git a/scripts/ruaccent-requirements.txt b/scripts/ruaccent-requirements.txt new file mode 100644 index 0000000..c0065b3 --- /dev/null +++ b/scripts/ruaccent-requirements.txt @@ -0,0 +1,25 @@ +certifi==2026.7.22 +charset-normalizer==3.5.1 +filelock==4.0.4 +flatbuffers==25.12.19 +fsspec==2026.9.0 +hf-xet==1.6.0 +huggingface-hub==0.36.2 +idna==3.20 +numpy==2.5.3 +onnxruntime==1.30.0 +packaging==26.3 +protobuf==7.36.2 +python-crfsuite==0.9.12 +pyyaml==6.0.3 +razdel==0.5.0 +regex==2026.9.10 +requests==2.34.2 +ruaccent==1.5.8.3 +safetensors==0.8.0 +sentencepiece==0.2.2 +tokenizers==0.22.2 +tqdm==4.70.1 +transformers==4.57.6 +typing-extensions==4.16.0 +urllib3==2.8.0 diff --git a/scripts/setup-ruaccent.py b/scripts/setup-ruaccent.py new file mode 100644 index 0000000..711c997 --- /dev/null +++ b/scripts/setup-ruaccent.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Workspace-local RuAccent CPU setup. No sudo or changes to system Python.""" +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import runpy +import shutil +import subprocess +import urllib.request +import zipfile + +ROOT = Path(__file__).resolve().parent.parent + + +def main(): + uv = shutil.which("uv") or str(ROOT / ".local/tools/uv") + if not Path(uv).is_file(): + if platform.system() != "Linux" or platform.machine() != "x86_64": + raise SystemExit("Install uv first on this platform, then rerun this setup script.") + with urllib.request.urlopen("https://pypi.org/pypi/uv/0.12.19/json", timeout=60) as source: + metadata = json.load(source) + wheel = next(x for x in metadata["urls"] if x["filename"].endswith("manylinux_2_17_x86_64.manylinux2014_x86_64.whl")) + with urllib.request.urlopen(wheel["url"], timeout=60) as source: + content = source.read() + expected = "a63d18a0aa38ee9f21a5406afbbaeb41303bcd954be9d6b7c1b95ac275e53958" + if hashlib.sha256(content).hexdigest() != expected: + raise RuntimeError("uv checksum mismatch") + with zipfile.ZipFile(io.BytesIO(content)) as archive: + name = next(x for x in archive.namelist() if x.endswith("/uv")) + Path(uv).parent.mkdir(parents=True, exist_ok=True) + Path(uv).write_bytes(archive.read(name)) + Path(uv).chmod(0o755) + environment = dict(os.environ, UV_CACHE_DIR=str(ROOT / ".local/uv-cache"), + UV_PYTHON_INSTALL_DIR=str(ROOT / ".local/python"), UV_SYSTEM_CERTS="true") + venv = ROOT / ".local/ruaccent-runtime" + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + if not python.is_file(): + subprocess.run([uv, "venv", "--python", "3.12.14", str(venv)], env=environment, check=True) + subprocess.run([uv, "pip", "sync", "--python", str(python), str(ROOT / "scripts/ruaccent-requirements.txt")], env=environment, check=True) + model = ROOT / ".local/tts/ruaccent" + manifest = json.loads((ROOT / "scripts/ruaccent-model.json").read_text()) + runpy.run_path(str(ROOT / "scripts/download-tts-model.py"))["download"](manifest, model) + # RUAccent 1.5.8.3 expects its rule-engine files beside the installed package. + module_path = subprocess.check_output([str(python), "-c", "import importlib.util; print(importlib.util.find_spec('ruaccent').submodule_search_locations[0])"], text=True).strip() + shutil.copytree(model / "koziev", Path(module_path) / "koziev", dirs_exist_ok=True) + worker = ROOT / "composeApp/src/jvmMain/resources/tts/ruaccent-worker.py" + shutil.copyfile(worker, model / "worker.py") + # Verify a wholly offline initialization before declaring the installation ready. + subprocess.run([str(python), str(model / "worker.py"), str(model), "--check"], check=True, env=environment) + (model / "runtime.json").write_text(json.dumps({"python": str(python), "model": "turbo3.1", "revision": manifest["revision"]}, indent=2) + "\n") + print("RuAccent ready: CPU, turbo3.1, full dictionary; no network during inference.", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/silero-requirements.txt b/scripts/silero-requirements.txt new file mode 100644 index 0000000..988591e --- /dev/null +++ b/scripts/silero-requirements.txt @@ -0,0 +1,10 @@ +filelock==3.32.3 +fsspec==2026.7.0 +jinja2==3.1.6 +markupsafe==3.0.3 +mpmath==1.3.0 +networkx==3.6.1 +setuptools==78.1.0 +sympy==1.14.0 +torch==2.8.0+cpu +typing-extensions==4.16.0 diff --git a/scripts/silero-sample.py b/scripts/silero-sample.py new file mode 100644 index 0000000..0558aaa --- /dev/null +++ b/scripts/silero-sample.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Audition official Silero v5_5_ru locally on CPU, without changing the reader.""" +import argparse +import array +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import wave + +ROOT = Path(__file__).resolve().parent.parent +os.environ["CUDA_VISIBLE_DEVICES"] = "" +os.environ.setdefault("TORCH_HOME", str(ROOT / ".local/torch")) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--voices", nargs="+", default=["xenia", "baya", "aidar"]) + args = parser.parse_args() + import torch + + torch.set_num_threads(4) + torch.set_num_interop_threads(1) + torch.manual_seed(42) + model_path = ROOT / ".local/tts/silero-v5_5-ru/model.pt" + output = ROOT / "composeApp/build/tts-samples" + output.mkdir(parents=True, exist_ok=True) + start = time.perf_counter() + model = torch.package.PackageImporter(str(model_path)).load_pickle("tts_models", "model") + model.to(torch.device("cpu")) + loaded = time.perf_counter() - start + print(f"Silero v5_5_ru, CPU, 4 threads; load {loaded:.3f}s; voices {model.speakers}", flush=True) + samples = { + "ru": "В библиотеке было тихо. Она открыла книгу и начала читать. За окном шёл осенний дождь. На двери висел старый замок, а вдали виднелся замок на холме. Завтра, подумала она, мы наконец доберёмся до моря.", + "ru-yo": "Елка росла у дома. Ежик шел по лесу. Ребенок пьет теплое молоко. Все уже пришли. Он прочитал все. Она все еще ждет. Он узнает об этом завтра. Он узнает ее по голосу.", + } + report = {"model": "v5_5_ru", "sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), + "torch": torch.__version__, "device": "cpu", "threads": 4, "load_seconds": loaded, + "put_accent": True, "put_yo": True, "put_stress_homo": True, "put_yo_homo": True, "samples": []} + for voice in args.voices: + for label, text in samples.items(): + start = time.perf_counter() + with torch.inference_mode(): + audio = model.apply_tts(text=text, speaker=voice, sample_rate=48000, + put_accent=True, put_yo=True, + put_stress_homo=True, put_yo_homo=True).detach().cpu().flatten() + elapsed = time.perf_counter() - start + assert audio.numel() and torch.isfinite(audio).all() and audio.abs().max() > 0.001 + duration = audio.numel() / 48000 + pcm = array.array("h", (audio.clamp(-1, 1) * 32767).round().to(torch.int16).tolist()) + if sys.byteorder != "little": + pcm.byteswap() + path = output / f"silero-{label}-{voice}.wav" + with wave.open(str(path), "wb") as wav: + wav.setparams((1, 2, 48000, 0, "NONE", "not compressed")) + wav.writeframes(pcm.tobytes()) + path.with_suffix(".txt").write_text(text + "\n") + report["samples"].append({"file": path.name, "synthesis_seconds": elapsed, + "audio_seconds": duration, "rtf": elapsed / duration}) + print(f"{path.name}: {elapsed:.3f}s synthesis, {duration:.3f}s audio, RTF {elapsed / duration:.3f}", flush=True) + (output / "silero-benchmark.json").write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/supertonic-3-model.json b/scripts/supertonic-3-model.json new file mode 100644 index 0000000..c2e8584 --- /dev/null +++ b/scripts/supertonic-3-model.json @@ -0,0 +1,91 @@ +{ + "repository": "supertone-oss-archive/supertonic-3", + "revision": "aafc6e32416a594460b32413efc49d7fe4ce6d46", + "files": [ + { + "path": "LICENSE", + "size": 15007, + "git_blob": "1e2cbe6cef94c8cdf3ed1fcebc0f5317ca7ad5a1" + }, + { + "path": "onnx/duration_predictor.onnx", + "size": 3700147, + "sha256": "c3eb91414d5ff8a7a239b7fe9e34e7e2bf8a8140d8375ffb14718b1c639325db" + }, + { + "path": "onnx/text_encoder.onnx", + "size": 36416150, + "sha256": "c7befd5ea8c3119769e8a6c1486c4edc6a3bc8365c67621c881bbb774b9902ff" + }, + { + "path": "onnx/tts.json", + "size": 8253, + "git_blob": "28575b0793c92607a0f0a292550df9bd17709a79" + }, + { + "path": "onnx/unicode_indexer.json", + "size": 277676, + "git_blob": "3aaf4d33024aff0be455633c24b3636b3a810150" + }, + { + "path": "onnx/vector_estimator.onnx", + "size": 256534781, + "sha256": "883ac868ea0275ef0e991524dc64f16b3c0376efd7c320af6b53f5b780d7c61c" + }, + { + "path": "onnx/vocoder.onnx", + "size": 101424195, + "sha256": "085de76dd8e8d5836d6ca66826601f615939218f90e519f70ee8a36ed2a4c4ba" + }, + { + "path": "voice_styles/F1.json", + "size": 292046, + "git_blob": "421365f307ed1535d8da16845031d0e9fa3e60c5" + }, + { + "path": "voice_styles/F2.json", + "size": 292423, + "git_blob": "b09a3a433df8a4856d1d7d64736a56ddf37a4ea3" + }, + { + "path": "voice_styles/F3.json", + "size": 290794, + "git_blob": "a366f94c77c4b440404071f1a52bed3ccf83a5ff" + }, + { + "path": "voice_styles/F4.json", + "size": 291808, + "git_blob": "39c78a52795320c6101b7eaf984c0e2352fbdc18" + }, + { + "path": "voice_styles/F5.json", + "size": 291479, + "git_blob": "06983f8816ad00a8eeb7efd37a33d457587e16f8" + }, + { + "path": "voice_styles/M1.json", + "size": 291748, + "git_blob": "bddfff85a7c4c140c11620bd7005d86681dccdac" + }, + { + "path": "voice_styles/M2.json", + "size": 292055, + "git_blob": "602ff6c4d6d48cdb71c37461f7c175d71fe8b34b" + }, + { + "path": "voice_styles/M3.json", + "size": 290198, + "git_blob": "000b60436b77619b933fdc33628ad737595e6794" + }, + { + "path": "voice_styles/M4.json", + "size": 291522, + "git_blob": "aae253ae81185fa06a6469c3f1cdff04ed121ddf" + }, + { + "path": "voice_styles/M5.json", + "size": 291469, + "git_blob": "0536b252698830f8536c077edcb94e1ce4768ab6" + } + ] +}