Add offline desktop speech with Silero Xenia for Russian
This commit is contained in:
1 parent
9cd8ebc422
commit
9dbe30b20c
28 files changed
+1865
-16
No files matched your search
@@ -96,10 +96,29 @@ kotlin {
|
||||
jvmMain.dependencies {
|
||||
implementation(compose.desktop.currentOs)
|
||||
implementation(libs.kotlinx.coroutinesSwing)
|
||||
implementation(libs.onnxruntime)
|
||||
implementation(libs.jackson.databind)
|
||||
}
|
||||
jvmTest.dependencies {
|
||||
implementation(libs.kotlinx.coroutinesTest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register<JavaExec>("ttsSample") {
|
||||
group = "verification"
|
||||
description = "Generate plain English/Russian WAV samples using the local Supertonic CPU model."
|
||||
dependsOn("jvmMainClasses")
|
||||
mainClass.set("net.sergeych.toread.tts.TtsSampleKt")
|
||||
classpath = configurations["jvmRuntimeClasspath"] + kotlin.targets["jvm"].compilations["main"].output.allOutputs
|
||||
workingDir = rootProject.projectDir
|
||||
args(layout.buildDirectory.dir("tts-samples").get().asFile.absolutePath)
|
||||
if (providers.gradleProperty("ttsPlay").orNull == "true") args("--play")
|
||||
if (providers.gradleProperty("ttsStress").orNull == "true") args("--stress")
|
||||
if (providers.gradleProperty("ttsRuAccent").orNull == "true") args("--ruaccent")
|
||||
if (providers.gradleProperty("ttsSilero").orNull == "true") args("--silero")
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "net.sergeych.toread"
|
||||
compileSdk = libs.versions.android.compileSdk.get().toInt()
|
||||
|
||||
@@ -174,6 +174,9 @@ internal open class AppStrings {
|
||||
open val offlineVoice = "Offline voice"
|
||||
open val autoRussian = "Auto Russian"
|
||||
open val loadingVoices = "Loading voices..."
|
||||
open val preparingSpeech = "Preparing speech…"
|
||||
open val offlineSpeechModelMissing = "Download the offline speech model first; see the desktop setup guide."
|
||||
open val speechPlaybackFailed = "Speech playback failed. Check the offline model and audio output, then try again."
|
||||
|
||||
open val imageCopied = "Image copied"
|
||||
open val copyFailed = "Copy failed"
|
||||
@@ -399,6 +402,9 @@ internal object RussianStrings : AppStrings() {
|
||||
override val offlineVoice = "Офлайн-голос"
|
||||
override val autoRussian = "Авто: русский"
|
||||
override val loadingVoices = "Загружаем голоса..."
|
||||
override val preparingSpeech = "Подготавливаем речь…"
|
||||
override val offlineSpeechModelMissing = "Сначала загрузите офлайн-модель речи по инструкции для компьютера."
|
||||
override val speechPlaybackFailed = "Не удалось воспроизвести речь. Проверьте офлайн-модель и вывод звука и попробуйте снова."
|
||||
|
||||
override val imageCopied = "Изображение скопировано"
|
||||
override val copyFailed = "Не удалось скопировать"
|
||||
|
||||
@@ -17,6 +17,7 @@ data class ReadAloudState(
|
||||
val active: Boolean = false,
|
||||
val playing: Boolean = false,
|
||||
val sentenceIndex: Int? = null,
|
||||
val buffering: Boolean = false,
|
||||
)
|
||||
|
||||
data class ReadAloudEngineOption(
|
||||
@@ -45,6 +46,7 @@ data class ReadAloudSettingsState(
|
||||
val textReplacements: List<ReadAloudTextReplacement> = emptyList(),
|
||||
val loading: Boolean = false,
|
||||
val message: String? = null,
|
||||
val usesSystemDefaults: Boolean = true,
|
||||
)
|
||||
|
||||
expect object ReadAloudPlatform {
|
||||
|
||||
@@ -204,6 +204,10 @@ internal fun BookView(
|
||||
}
|
||||
}
|
||||
|
||||
LaunchedEffect(readAloudSettings.message) {
|
||||
readAloudSettings.message?.let { snackbarHostState.showSnackbar(it) }
|
||||
}
|
||||
|
||||
fun openReaderLink(href: String) {
|
||||
if (href.isWebUrl()) {
|
||||
openPlatformUrl(href)
|
||||
@@ -549,6 +553,7 @@ internal fun BookView(
|
||||
if (readAloudPanelVisible && readAloudState.active) {
|
||||
ReadAloudPanel(
|
||||
playing = readAloudState.playing,
|
||||
buffering = readAloudState.buffering,
|
||||
onPlayStop = {
|
||||
if (readAloudState.playing) {
|
||||
ReadAloudPlatform.stop()
|
||||
@@ -1626,6 +1631,7 @@ private val DarkReaderTextPalette = listOf(
|
||||
@Composable
|
||||
private fun ReadAloudPanel(
|
||||
playing: Boolean,
|
||||
buffering: Boolean,
|
||||
onPlayStop: () -> Unit,
|
||||
onBack: () -> Unit,
|
||||
onForward: () -> Unit,
|
||||
@@ -1642,6 +1648,10 @@ private fun ReadAloudPanel(
|
||||
horizontalArrangement = Arrangement.Center,
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
if (buffering) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(20.dp))
|
||||
Text(strings.preparingSpeech, modifier = Modifier.padding(horizontal = 8.dp))
|
||||
}
|
||||
IconButton(onClick = onBack) {
|
||||
Icon(Icons.Filled.Replay, contentDescription = strings.previousSentence)
|
||||
}
|
||||
@@ -1686,7 +1696,7 @@ private fun ReadAloudSettingsDialog(
|
||||
Text(selectedEngineLabel)
|
||||
}
|
||||
DropdownMenu(expanded = engineMenuOpen, onDismissRequest = { engineMenuOpen = false }) {
|
||||
DropdownMenuItem(
|
||||
if (state.usesSystemDefaults) DropdownMenuItem(
|
||||
text = { Text(strings.systemDefault) },
|
||||
onClick = {
|
||||
engineMenuOpen = false
|
||||
@@ -1711,7 +1721,7 @@ private fun ReadAloudSettingsDialog(
|
||||
Text(selectedVoiceLabel)
|
||||
}
|
||||
DropdownMenu(expanded = voiceMenuOpen, onDismissRequest = { voiceMenuOpen = false }) {
|
||||
DropdownMenuItem(
|
||||
if (state.usesSystemDefaults) DropdownMenuItem(
|
||||
text = { Text(strings.autoRussian) },
|
||||
onClick = {
|
||||
voiceMenuOpen = false
|
||||
|
||||
@@ -1,20 +1,54 @@
|
||||
package net.sergeych.toread
|
||||
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import net.sergeych.toread.tts.DesktopReadAloudPlayer
|
||||
import net.sergeych.toread.tts.SpeechAudio
|
||||
import net.sergeych.toread.tts.RuAccentProcess
|
||||
import net.sergeych.toread.tts.SileroProcess
|
||||
import net.sergeych.toread.tts.SupertonicTts
|
||||
import net.sergeych.toread.tts.playDesktopAudio
|
||||
|
||||
actual object ReadAloudPlatform {
|
||||
actual val isSupported: Boolean = false
|
||||
private val mutableState = MutableStateFlow(ReadAloudState())
|
||||
actual val state: StateFlow<ReadAloudState> = mutableState
|
||||
private val mutableSettingsState = MutableStateFlow(ReadAloudSettingsState())
|
||||
actual val settingsState: StateFlow<ReadAloudSettingsState> = mutableSettingsState
|
||||
actual val isSupported: Boolean = true
|
||||
private val inferenceLock = Mutex()
|
||||
private var synthesizer: SupertonicTts? = null
|
||||
private val accentor = if (RuAccentProcess.isInstalled()) RuAccentProcess() else null
|
||||
private val silero = if (SileroProcess.isInstalled()) SileroProcess() else null
|
||||
private val player = DesktopReadAloudPlayer(
|
||||
scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate),
|
||||
synthesize = { text, voice ->
|
||||
withContext(Dispatchers.IO) {
|
||||
inferenceLock.withLock {
|
||||
val context = currentCoroutineContext()
|
||||
context.ensureActive()
|
||||
val engine = synthesizer ?: SupertonicTts().also { synthesizer = it }
|
||||
context.ensureActive()
|
||||
SpeechAudio(engine.synthesize(text, voice, checkCancelled = { context.ensureActive() }), engine.sampleRate)
|
||||
}
|
||||
}
|
||||
},
|
||||
playAudio = ::playDesktopAudio,
|
||||
installed = { SupertonicTts.isInstalled() },
|
||||
accentuate = accentor?.let { processor -> { text -> processor.accentuate(text) } },
|
||||
silero = silero?.let { processor -> { text, voice -> processor.synthesize(text, voice) } },
|
||||
)
|
||||
actual val state: StateFlow<ReadAloudState> = player.state
|
||||
actual val settingsState: StateFlow<ReadAloudSettingsState> = player.settingsState
|
||||
|
||||
actual fun prepare(fileId: String, bookTitle: String, sentences: List<ReadAloudSentence>, startIndex: Int) = Unit
|
||||
actual fun play() = Unit
|
||||
actual fun stop() = Unit
|
||||
actual fun skip(delta: Int) = Unit
|
||||
actual fun refreshSettings() = Unit
|
||||
actual fun selectEngine(engineId: String?) = Unit
|
||||
actual fun selectVoice(voiceId: String?) = Unit
|
||||
actual fun prepare(fileId: String, bookTitle: String, sentences: List<ReadAloudSentence>, startIndex: Int) =
|
||||
player.prepare(sentences, startIndex)
|
||||
actual fun play() = player.play()
|
||||
actual fun stop() = player.stop()
|
||||
actual fun skip(delta: Int) = player.skip(delta)
|
||||
actual fun refreshSettings() = player.refreshSettings()
|
||||
actual fun selectEngine(engineId: String?) = player.selectEngine(engineId)
|
||||
actual fun selectVoice(voiceId: String?) = player.selectVoice(voiceId)
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.File
|
||||
import javax.sound.sampled.AudioFileFormat
|
||||
import javax.sound.sampled.AudioFormat
|
||||
import javax.sound.sampled.AudioInputStream
|
||||
import javax.sound.sampled.AudioSystem
|
||||
import javax.sound.sampled.LineEvent
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
internal fun SpeechAudio.pcm16(): ByteArray = ByteArray(samples.size * 2).also { bytes ->
|
||||
samples.forEachIndexed { index, sample ->
|
||||
val pcm = (sample.takeIf { it.isFinite() }?.coerceIn(-1f, 1f)?.times(32767) ?: 0f).toInt()
|
||||
bytes[index * 2] = pcm.toByte()
|
||||
bytes[index * 2 + 1] = (pcm shr 8).toByte()
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun playDesktopAudio(audio: SpeechAudio) = withContext(Dispatchers.IO) {
|
||||
val format = AudioFormat(audio.sampleRate.toFloat(), 16, 1, true, false)
|
||||
val pcm = audio.pcm16()
|
||||
val clip = AudioSystem.getClip()
|
||||
try {
|
||||
currentCoroutineContext().ensureActive()
|
||||
clip.open(format, pcm, 0, pcm.size)
|
||||
suspendCancellableCoroutine<Unit> { continuation ->
|
||||
val completed = AtomicBoolean(false)
|
||||
clip.addLineListener { event ->
|
||||
if (event.type == LineEvent.Type.STOP && completed.compareAndSet(false, true)) {
|
||||
continuation.resume(Unit)
|
||||
}
|
||||
}
|
||||
continuation.invokeOnCancellation {
|
||||
clip.stop()
|
||||
clip.close()
|
||||
}
|
||||
if (continuation.isActive) clip.start()
|
||||
}
|
||||
} finally {
|
||||
clip.stop()
|
||||
clip.close()
|
||||
}
|
||||
}
|
||||
|
||||
internal fun SpeechAudio.writeWav(file: File) {
|
||||
file.parentFile?.mkdirs()
|
||||
val format = AudioFormat(sampleRate.toFloat(), 16, 1, true, false)
|
||||
AudioInputStream(ByteArrayInputStream(pcm16()), format, samples.size.toLong()).use {
|
||||
AudioSystem.write(it, AudioFileFormat.Type.WAVE, file)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun joinSpeechAudio(parts: List<SpeechAudio>): SpeechAudio {
|
||||
require(parts.isNotEmpty()) { "No speech to play" }
|
||||
if (parts.size == 1) return parts.single()
|
||||
val rate = parts.first().sampleRate
|
||||
require(parts.all { it.sampleRate == rate }) { "Mismatched speech sample rates" }
|
||||
val pause = rate / 10
|
||||
val samples = FloatArray(parts.sumOf { it.samples.size } + pause * (parts.size - 1))
|
||||
var offset = 0
|
||||
parts.forEach { part -> part.samples.copyInto(samples, offset); offset += part.samples.size + pause }
|
||||
return SpeechAudio(samples, rate)
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import net.sergeych.toread.ReadAloudEngineOption
|
||||
import net.sergeych.toread.ReadAloudSentence
|
||||
import net.sergeych.toread.ReadAloudSettingsState
|
||||
import net.sergeych.toread.ReadAloudState
|
||||
import net.sergeych.toread.ReadAloudVoiceOption
|
||||
import net.sergeych.toread.strings
|
||||
|
||||
internal data class SpeechAudio(val samples: FloatArray, val sampleRate: Int)
|
||||
|
||||
/** Commands and state changes run on the UI dispatcher; inference/audio run off-thread.
|
||||
* A playback job owns its lookahead, so stop/skip/voice changes cannot publish stale audio.
|
||||
*/
|
||||
internal class DesktopReadAloudPlayer(
|
||||
private val scope: CoroutineScope,
|
||||
private val synthesize: suspend (String, String) -> SpeechAudio,
|
||||
private val playAudio: suspend (SpeechAudio) -> Unit,
|
||||
private val installed: () -> Boolean,
|
||||
private val accentuate: (suspend (String) -> String)? = null,
|
||||
private val silero: (suspend (String, String) -> SpeechAudio)? = null,
|
||||
) {
|
||||
private fun voices(engine: String) = if (engine == "silero")
|
||||
SileroProcess.voices.map { ReadAloudVoiceOption(it, it.replaceFirstChar(Char::uppercase), "ru (English: F1)", false) }
|
||||
else SupertonicTts.voices.map { ReadAloudVoiceOption(it, it, "en / ru", false) }
|
||||
private val defaultEngine = if (silero != null) "silero" else if (accentuate != null) "supertonic-3-ruaccent" else "supertonic-3"
|
||||
private val mutableState = MutableStateFlow(ReadAloudState())
|
||||
val state = mutableState.asStateFlow()
|
||||
private val mutableSettings = MutableStateFlow(ReadAloudSettingsState(
|
||||
engines = (if (silero != null) listOf(ReadAloudEngineOption("silero", "Silero (RU) / Supertonic (EN)")) else emptyList()) +
|
||||
listOf(ReadAloudEngineOption("supertonic-3", "Supertonic 3")) +
|
||||
if (accentuate != null) listOf(ReadAloudEngineOption("supertonic-3-ruaccent", "Supertonic 3 + RUAccent")) else emptyList(),
|
||||
selectedEngineId = defaultEngine,
|
||||
voices = voices(defaultEngine),
|
||||
selectedVoiceId = if (silero != null) "xenia" else "F1",
|
||||
usesSystemDefaults = false,
|
||||
))
|
||||
val settingsState = mutableSettings.asStateFlow()
|
||||
private var queue = emptyList<ReadAloudSentence>()
|
||||
private var index = 0
|
||||
private var playback: Job? = null
|
||||
|
||||
fun prepare(sentences: List<ReadAloudSentence>, startIndex: Int) {
|
||||
stop()
|
||||
queue = sentences.toList()
|
||||
index = if (queue.isEmpty()) 0 else startIndex.coerceIn(queue.indices)
|
||||
mutableState.value = ReadAloudState(active = queue.isNotEmpty(), sentenceIndex = queue.getOrNull(index)?.index)
|
||||
}
|
||||
|
||||
fun play() {
|
||||
if (queue.isEmpty()) return
|
||||
playback?.cancel()
|
||||
mutableSettings.value = mutableSettings.value.copy(message = null)
|
||||
if (!installed()) {
|
||||
mutableState.value = mutableState.value.copy(active = true, playing = false, buffering = false)
|
||||
mutableSettings.value = mutableSettings.value.copy(message = strings.offlineSpeechModelMissing)
|
||||
return
|
||||
}
|
||||
val voice = mutableSettings.value.selectedVoiceId ?: "F1"
|
||||
val useSilero = mutableSettings.value.selectedEngineId == "silero"
|
||||
val useAccents = mutableSettings.value.selectedEngineId == "supertonic-3-ruaccent"
|
||||
suspend fun render(text: String): SpeechAudio {
|
||||
if (useSilero) return if (SupertonicTts.languageOf(text) == "ru") silero!!.invoke(text, voice)
|
||||
else synthesize(text, "F1")
|
||||
val prepared = if (useAccents) accentuate!!.invoke(text) else text
|
||||
// Accent marks increase character count; re-chunk before entering the TTS model.
|
||||
return joinSpeechAudio(SupertonicTts.chunks(SupertonicTts.russianStressToUnicode(prepared)).map { synthesize(it, voice) })
|
||||
}
|
||||
val chunks = queue.asSequence().withIndex().drop(index).flatMap { (position, sentence) ->
|
||||
val parts = SupertonicTts.chunks(sentence.spokenText)
|
||||
parts.asSequence().mapIndexed { chunkIndex, text ->
|
||||
Chunk(position, sentence, text, chunkIndex == 0, chunkIndex == parts.lastIndex)
|
||||
}
|
||||
}.iterator()
|
||||
mutableState.value = mutableState.value.copy(active = true, playing = true, buffering = true)
|
||||
playback = scope.launch {
|
||||
try {
|
||||
coroutineScope {
|
||||
// Keep at most one audio chunk ahead of the currently playing chunk.
|
||||
var next = if (chunks.hasNext()) chunks.next() else null
|
||||
var pending = next?.let { chunk -> async { render(chunk.text) } }
|
||||
while (next != null) {
|
||||
val current = next
|
||||
mutableState.value = mutableState.value.copy(buffering = pending?.isCompleted != true)
|
||||
val audio = pending!!.await()
|
||||
next = if (chunks.hasNext()) chunks.next() else null
|
||||
pending = next?.let { chunk -> async { render(chunk.text) } }
|
||||
index = current.position
|
||||
mutableState.value = ReadAloudState(active = true, playing = true, sentenceIndex = current.sentence.index)
|
||||
if (current.first) delay(current.sentence.pauseBeforeMillis)
|
||||
playAudio(audio)
|
||||
if (current.last) delay(current.sentence.pauseAfterMillis)
|
||||
}
|
||||
}
|
||||
mutableState.value = ReadAloudState()
|
||||
} catch (cancelled: CancellationException) {
|
||||
throw cancelled
|
||||
} catch (error: Exception) {
|
||||
System.err.println("Desktop TTS failed: ${error.javaClass.simpleName}: ${error.message}")
|
||||
mutableState.value = mutableState.value.copy(playing = false, buffering = false)
|
||||
mutableSettings.value = mutableSettings.value.copy(message = strings.speechPlaybackFailed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
playback?.cancel()
|
||||
playback = null
|
||||
mutableState.value = ReadAloudState()
|
||||
}
|
||||
|
||||
fun skip(delta: Int) {
|
||||
if (queue.isEmpty()) return
|
||||
val wasPlaying = mutableState.value.playing
|
||||
playback?.cancel()
|
||||
index = (index.toLong() + delta).coerceIn(0, queue.lastIndex.toLong()).toInt()
|
||||
mutableState.value = ReadAloudState(active = true, sentenceIndex = queue[index].index)
|
||||
if (wasPlaying) play()
|
||||
}
|
||||
|
||||
fun selectVoice(id: String?) {
|
||||
val voice = id ?: if (mutableSettings.value.selectedEngineId == "silero") "xenia" else "F1"
|
||||
if (mutableSettings.value.voices.none { it.id == voice } || voice == mutableSettings.value.selectedVoiceId) return
|
||||
mutableSettings.value = mutableSettings.value.copy(selectedVoiceId = voice)
|
||||
if (mutableState.value.playing) play()
|
||||
}
|
||||
|
||||
fun selectEngine(id: String?) {
|
||||
val engine = id ?: defaultEngine
|
||||
if (mutableSettings.value.engines.none { it.id == engine } || engine == mutableSettings.value.selectedEngineId) return
|
||||
val options = voices(engine)
|
||||
mutableSettings.value = mutableSettings.value.copy(selectedEngineId = engine, voices = options,
|
||||
selectedVoiceId = mutableSettings.value.selectedVoiceId?.takeIf { id -> options.any { it.id == id } }
|
||||
?: if (engine == "silero") "xenia" else "F1")
|
||||
if (mutableState.value.playing) play()
|
||||
}
|
||||
|
||||
fun refreshSettings() {
|
||||
mutableSettings.value = mutableSettings.value.copy(message = if (installed()) null else strings.offlineSpeechModelMissing)
|
||||
}
|
||||
|
||||
private data class Chunk(
|
||||
val position: Int,
|
||||
val sentence: ReadAloudSentence,
|
||||
val text: String,
|
||||
val first: Boolean,
|
||||
val last: Boolean,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import java.io.BufferedReader
|
||||
import java.io.BufferedWriter
|
||||
import java.io.File
|
||||
|
||||
/** Optional desktop-only CPU accentor. Persistent local process, no HTTP or remote text service. */
|
||||
internal class RuAccentProcess(private val directory: File = installationDirectory()) : AutoCloseable {
|
||||
private val mapper = ObjectMapper()
|
||||
private val lock = Mutex()
|
||||
@Volatile private var worker: Worker? = null
|
||||
|
||||
init {
|
||||
Runtime.getRuntime().addShutdownHook(Thread { close() })
|
||||
}
|
||||
|
||||
suspend fun accentuate(text: String): String {
|
||||
// Respect explicitly supplied stress; English needs no Russian processor.
|
||||
if (SupertonicTts.languageOf(text) == "en" || '\u0301' in text || Regex("\\+[аеёиоуыэюяАЕЁИОУЫЭЮЯ]").containsMatchIn(text)) return text
|
||||
return lock.withLock {
|
||||
try {
|
||||
currentCoroutineContext().ensureActive()
|
||||
val current = worker?.takeIf { it.process.isAlive } ?: startWorker().also { worker = it }
|
||||
withTimeout(90_000) {
|
||||
coroutineScope {
|
||||
val response = async(Dispatchers.IO) {
|
||||
if (!current.ready) {
|
||||
val ready = current.output.readLine() ?: error("RuAccent exited while loading")
|
||||
check(mapper.readTree(ready)["ready"]?.asBoolean() == true) { "RuAccent did not initialize" }
|
||||
current.ready = true
|
||||
}
|
||||
current.input.write(mapper.writeValueAsString(mapOf("text" to text)))
|
||||
current.input.newLine()
|
||||
current.input.flush()
|
||||
val line = current.output.readLine() ?: error("RuAccent exited during inference")
|
||||
val result = mapper.readTree(line)
|
||||
check(result["error"] == null) { "RuAccent inference failed" }
|
||||
result["text"]?.asText()?.takeIf { it.isNotBlank() } ?: error("Empty RuAccent response")
|
||||
}
|
||||
try {
|
||||
response.await()
|
||||
} finally {
|
||||
// A blocked pipe read cannot be interrupted reliably; killing the
|
||||
// cancelled worker unblocks it before the next request acquires the lock.
|
||||
if (!isActive) current.process.destroyForcibly()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: Exception) {
|
||||
close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun startWorker(): Worker {
|
||||
val runtime = mapper.readTree(File(directory, "runtime.json"))
|
||||
val process = ProcessBuilder(runtime["python"].asText(), "-u", File(directory, "worker.py").path, directory.path)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.apply {
|
||||
environment()["HF_HUB_OFFLINE"] = "1"
|
||||
environment()["TRANSFORMERS_OFFLINE"] = "1"
|
||||
environment()["HF_HUB_DISABLE_TELEMETRY"] = "1"
|
||||
}.start()
|
||||
return Worker(process, process.inputStream.bufferedReader(Charsets.UTF_8), process.outputStream.bufferedWriter(Charsets.UTF_8))
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
worker?.process?.destroyForcibly()
|
||||
worker = null
|
||||
}
|
||||
|
||||
private class Worker(val process: Process, val output: BufferedReader, val input: BufferedWriter, var ready: Boolean = false)
|
||||
|
||||
companion object {
|
||||
fun installationDirectory(): File = System.getenv("YEREED_RUACCENT_HOME")?.let(::File)
|
||||
?: File(SupertonicTts.modelDirectory().parentFile, "ruaccent")
|
||||
fun isInstalled(): Boolean = File(installationDirectory(), "runtime.json").isFile && File(installationDirectory(), "worker.py").isFile
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import java.io.BufferedReader
|
||||
import java.io.BufferedWriter
|
||||
import java.io.File
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.Base64
|
||||
|
||||
/** Desktop-only CPU Silero synthesizer. Persistent local process, no HTTP or remote text service. */
|
||||
internal class SileroProcess(private val directory: File = installationDirectory()) : AutoCloseable {
|
||||
private val mapper = ObjectMapper()
|
||||
private val lock = Mutex()
|
||||
@Volatile private var worker: Worker? = null
|
||||
|
||||
init {
|
||||
Runtime.getRuntime().addShutdownHook(Thread { close() })
|
||||
}
|
||||
|
||||
suspend fun synthesize(text: String, voice: String): SpeechAudio {
|
||||
return lock.withLock {
|
||||
try {
|
||||
currentCoroutineContext().ensureActive()
|
||||
val current = worker?.takeIf { it.process.isAlive } ?: startWorker().also { worker = it }
|
||||
withTimeout(90_000) {
|
||||
coroutineScope {
|
||||
val response = async(Dispatchers.IO) {
|
||||
if (!current.ready) {
|
||||
val ready = current.output.readLine() ?: error("Silero exited while loading")
|
||||
check(mapper.readTree(ready)["ready"]?.asBoolean() == true) { "Silero did not initialize" }
|
||||
current.ready = true
|
||||
}
|
||||
current.input.write(mapper.writeValueAsString(mapOf("text" to text, "voice" to voice)))
|
||||
current.input.newLine()
|
||||
current.input.flush()
|
||||
val line = current.output.readLine() ?: error("Silero exited during inference")
|
||||
val result = mapper.readTree(line)
|
||||
check(result["error"] == null) { "Silero inference failed" }
|
||||
val bytes = Base64.getDecoder().decode(result["audio"].asText())
|
||||
check(bytes.isNotEmpty() && bytes.size % 4 == 0) { "Invalid Silero audio" }
|
||||
val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
|
||||
val samples = FloatArray(buffer.remaining()).also { buffer.get(it) }
|
||||
check(samples.all { it.isFinite() }) { "Non-finite Silero audio" }
|
||||
SpeechAudio(samples, result["sampleRate"].asInt().also { check(it == 48000) })
|
||||
}
|
||||
try {
|
||||
response.await()
|
||||
} finally {
|
||||
// A blocked pipe read cannot be interrupted reliably; killing the
|
||||
// cancelled worker unblocks it before the next request acquires the lock.
|
||||
if (!isActive) current.process.destroyForcibly()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: Exception) {
|
||||
close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun startWorker(): Worker {
|
||||
val runtime = mapper.readTree(File(directory, "runtime.json"))
|
||||
val script = SileroProcess::class.java.getResourceAsStream("/tts/silero-worker.py")!!.use { it.reader().readText() }
|
||||
val process = ProcessBuilder(runtime["python"].asText(), "-u", "-c", script, directory.path)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.apply {
|
||||
environment()["HF_HUB_OFFLINE"] = "1"
|
||||
environment()["TRANSFORMERS_OFFLINE"] = "1"
|
||||
environment()["HF_HUB_DISABLE_TELEMETRY"] = "1"
|
||||
}.start()
|
||||
return Worker(process, process.inputStream.bufferedReader(Charsets.UTF_8), process.outputStream.bufferedWriter(Charsets.UTF_8))
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
worker?.process?.destroyForcibly()
|
||||
worker = null
|
||||
}
|
||||
|
||||
private class Worker(val process: Process, val output: BufferedReader, val input: BufferedWriter, var ready: Boolean = false)
|
||||
|
||||
companion object {
|
||||
val voices = listOf("xenia", "baya", "aidar", "kseniya", "eugene")
|
||||
fun installationDirectory(): File = System.getenv("YEREED_SILERO_HOME")?.let(::File)
|
||||
?: File(SupertonicTts.modelDirectory().parentFile, "silero-v5_5-ru")
|
||||
fun isInstalled(): Boolean = File(installationDirectory(), "runtime.json").isFile && File(installationDirectory(), "model.pt").length() > 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import ai.onnxruntime.OnnxTensor
|
||||
import ai.onnxruntime.OrtEnvironment
|
||||
import ai.onnxruntime.OrtSession
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import java.io.File
|
||||
import java.nio.FloatBuffer
|
||||
import java.text.Normalizer
|
||||
import java.util.Random
|
||||
|
||||
/** CPU-only, single-utterance adaptation of Supertone's MIT Java example.
|
||||
* See docs/licenses/supertonic-code-MIT.txt. Call from one worker at a time.
|
||||
*/
|
||||
internal class SupertonicTts(val directory: File = modelDirectory()) : AutoCloseable {
|
||||
private val mapper = ObjectMapper()
|
||||
private val environment = OrtEnvironment.getEnvironment()
|
||||
private val config = mapper.readTree(File(directory, "onnx/tts.json"))
|
||||
private val indexer = mapper.readTree(File(directory, "onnx/unicode_indexer.json"))
|
||||
.map { it.asLong() }.toLongArray()
|
||||
val sampleRate: Int = config["ae"]["sample_rate"].asInt()
|
||||
private val compression = config["ttl"]["chunk_compress_factor"].asInt()
|
||||
private val chunkSize = config["ae"]["base_chunk_size"].asInt() * compression
|
||||
private val latentDimension = config["ttl"]["latent_dim"].asInt() * compression
|
||||
private val sessions = mutableListOf<OrtSession>()
|
||||
private val styles = mutableMapOf<String, JsonNode>()
|
||||
|
||||
init {
|
||||
try {
|
||||
OrtSession.SessionOptions().use { options ->
|
||||
// The standard (non-GPU) ONNX artifact uses CPUExecutionProvider only.
|
||||
options.setIntraOpNumThreads(4.coerceAtMost(Runtime.getRuntime().availableProcessors()))
|
||||
options.setInterOpNumThreads(1)
|
||||
for (name in listOf("duration_predictor", "text_encoder", "vector_estimator", "vocoder")) {
|
||||
sessions += environment.createSession(File(directory, "onnx/$name.onnx").path, options)
|
||||
}
|
||||
}
|
||||
} catch (error: Throwable) {
|
||||
close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
fun synthesize(
|
||||
text: String,
|
||||
voice: String = "F1",
|
||||
language: String = languageOf(text),
|
||||
seed: Long? = null,
|
||||
checkCancelled: () -> Unit = {},
|
||||
): FloatArray {
|
||||
require(voice in voices) { "Unknown voice: $voice" }
|
||||
require(language in listOf("en", "ru", "na")) { "Unsupported language: $language" }
|
||||
require(text.isNotBlank() && text.length <= MAX_CHARS) { "Expected 1–$MAX_CHARS characters" }
|
||||
checkCancelled()
|
||||
val prepared = prepareText(text, language)
|
||||
// Unknown symbols become spaces, never invalid (-1) embedding indices.
|
||||
val ids = prepared.codePoints().toArray().map { point ->
|
||||
indexer.getOrNull(point)?.takeIf { it >= 0 } ?: indexer[' '.code]
|
||||
}.toLongArray()
|
||||
val style = styles.getOrPut(voice) { mapper.readTree(File(directory, "voice_styles/$voice.json")) }
|
||||
NativeResources().use { resources ->
|
||||
fun tensor(value: Any) = resources.keep(OnnxTensor.createTensor(environment, value))
|
||||
fun styleTensor(name: String): OnnxTensor {
|
||||
val node = style[name]
|
||||
val values = node["data"].flatMap { batch -> batch.flatMap { row -> row.map { it.floatValue() } } }
|
||||
return resources.keep(OnnxTensor.createTensor(
|
||||
environment, FloatBuffer.wrap(values.toFloatArray()), node["dims"].map { it.asLong() }.toLongArray(),
|
||||
))
|
||||
}
|
||||
val textIds = tensor(arrayOf(ids))
|
||||
val textMask = tensor(arrayOf(arrayOf(FloatArray(ids.size) { 1f })))
|
||||
val styleDp = styleTensor("style_dp")
|
||||
val styleTtl = styleTensor("style_ttl")
|
||||
val duration = sessions[0].run(mapOf(
|
||||
"text_ids" to textIds, "text_mask" to textMask, "style_dp" to styleDp,
|
||||
)).use { result -> (result[0] as OnnxTensor).floatBuffer.get(0) / 1.05f }
|
||||
check(duration.isFinite() && duration > 0 && duration <= 60) { "Invalid speech duration: $duration" }
|
||||
checkCancelled()
|
||||
val sampleCount = (duration * sampleRate).toInt()
|
||||
val latentLength = (sampleCount + chunkSize - 1) / chunkSize
|
||||
val random = seed?.let(::Random) ?: Random()
|
||||
var latent = arrayOf(Array(latentDimension) { FloatArray(latentLength) { random.nextGaussian().toFloat() } })
|
||||
val latentMask = tensor(arrayOf(arrayOf(FloatArray(latentLength) { 1f })))
|
||||
val totalSteps = tensor(floatArrayOf(8f))
|
||||
val encoded = resources.keep(sessions[1].run(mapOf(
|
||||
"text_ids" to textIds, "text_mask" to textMask, "style_ttl" to styleTtl,
|
||||
)))
|
||||
val embedding = encoded[0] as OnnxTensor
|
||||
repeat(8) { step ->
|
||||
checkCancelled()
|
||||
OnnxTensor.createTensor(environment, latent).use { noisy ->
|
||||
OnnxTensor.createTensor(environment, floatArrayOf(step.toFloat())).use { currentStep ->
|
||||
sessions[2].run(mapOf(
|
||||
"noisy_latent" to noisy, "text_emb" to embedding, "style_ttl" to styleTtl,
|
||||
"latent_mask" to latentMask, "text_mask" to textMask,
|
||||
"current_step" to currentStep, "total_step" to totalSteps,
|
||||
)).use { result ->
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
latent = result[0].value as Array<Array<FloatArray>>
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
checkCancelled()
|
||||
OnnxTensor.createTensor(environment, latent).use { finalLatent ->
|
||||
return sessions[3].run(mapOf("latent" to finalLatent)).use { result ->
|
||||
val samples = (result[0] as OnnxTensor).floatBuffer
|
||||
check(samples.remaining() >= sampleCount) { "Truncated speech waveform" }
|
||||
FloatArray(sampleCount).also { samples.get(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
sessions.asReversed().forEach { runCatching { it.close() } }
|
||||
sessions.clear()
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val MAX_CHARS = 300
|
||||
val voices = listOf("F1", "F2", "F3", "F4", "F5", "M1", "M2", "M3", "M4", "M5")
|
||||
|
||||
fun modelDirectory(): File {
|
||||
val override = System.getProperty("yereed.tts.modelDir") ?: System.getenv("YEREED_TTS_MODEL_DIR")
|
||||
if (!override.isNullOrBlank()) return File(override)
|
||||
generateSequence(File(System.getProperty("user.dir")).absoluteFile) { it.parentFile }.forEach { root ->
|
||||
File(root, ".local/tts/supertonic-3").takeIf { it.isDirectory }?.let { return it }
|
||||
}
|
||||
return File(System.getProperty("user.home"), ".toread/tts/supertonic-3")
|
||||
}
|
||||
|
||||
fun isInstalled(directory: File = modelDirectory()): Boolean =
|
||||
(listOf("onnx/tts.json", "onnx/unicode_indexer.json") +
|
||||
listOf("duration_predictor", "text_encoder", "vector_estimator", "vocoder").map { "onnx/$it.onnx" } +
|
||||
voices.map { "voice_styles/$it.json" }).all { File(directory, it).length() > 0 }
|
||||
|
||||
fun languageOf(text: String): String {
|
||||
val russian = text.any { it in 'А'..'я' || it == 'ё' || it == 'Ё' }
|
||||
val english = text.any { it in 'a'..'z' || it in 'A'..'Z' }
|
||||
return when {
|
||||
russian && english -> "na"
|
||||
russian -> "ru"
|
||||
else -> "en"
|
||||
}
|
||||
}
|
||||
|
||||
/** Bounded chunks preserve every non-whitespace character, including long unbroken words. */
|
||||
fun chunks(text: String): List<String> = buildList {
|
||||
var remaining = text.trim()
|
||||
while (remaining.length > MAX_CHARS) {
|
||||
var end = remaining.lastIndexOfAny(charArrayOf(' ', '\n', '\t'), MAX_CHARS)
|
||||
if (end <= 0) end = MAX_CHARS
|
||||
if (Character.isHighSurrogate(remaining[end - 1])) end--
|
||||
while (end > 0 && Character.getType(remaining[end]) == Character.NON_SPACING_MARK.toInt()) end--
|
||||
require(end > 0) { "Oversized combining-character sequence" }
|
||||
add(remaining.substring(0, end).trim())
|
||||
remaining = remaining.substring(end).trimStart()
|
||||
}
|
||||
if (remaining.isNotEmpty()) add(remaining)
|
||||
}
|
||||
|
||||
internal fun prepareText(text: String, language: String): String {
|
||||
var normalized = Normalizer.normalize(russianStressToUnicode(text), Normalizer.Form.NFKD)
|
||||
.replace(Regex("[–‑—]"), "-")
|
||||
.replace(Regex("[“”]"), "\"")
|
||||
.replace(Regex("[‘’´`]"), "'")
|
||||
.replace(Regex("[_\\[\\]|/#→←<>]"), " ")
|
||||
.replace(Regex("[♥☆♡©\\\\]"), "")
|
||||
.replace(Regex("[\\x{1F000}-\\x{1FAFF}\\x{2600}-\\x{27BF}]"), "")
|
||||
if (language == "en") normalized = normalized.replace("@", " at ")
|
||||
.replace("e.g.,", "for example, ").replace("i.e.,", "that is, ")
|
||||
normalized = normalized.replace(Regex("\\s+"), " ").trim()
|
||||
.replace(Regex(" +([,.!?;:'])"), "$1")
|
||||
if (normalized.isEmpty() || normalized.last() !in ".!?;:,'\")]}…»") normalized += "."
|
||||
return "<$language>$normalized</$language>"
|
||||
}
|
||||
|
||||
/** RUAccent/Silero notation -> Cyrillic vowel plus U+0301, never a Latin lookalike. */
|
||||
internal fun russianStressToUnicode(text: String): String =
|
||||
text.replace(Regex("\\+([аеёиоуыэюяАЕЁИОУЫЭЮЯ])")) { it.groupValues[1] + "\u0301" }
|
||||
}
|
||||
}
|
||||
|
||||
private class NativeResources : AutoCloseable {
|
||||
private val values = mutableListOf<AutoCloseable>()
|
||||
fun <T : AutoCloseable> keep(value: T): T = value.also { values += it }
|
||||
override fun close() = values.asReversed().forEach { runCatching { it.close() } }
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.File
|
||||
import java.util.Locale
|
||||
import kotlin.system.measureNanoTime
|
||||
|
||||
/** Reproducible audition without opening the library, altering books, or contacting a service. */
|
||||
fun main(args: Array<String>) = runBlocking {
|
||||
val output = File(args.firstOrNull() ?: "build/tts-samples")
|
||||
if ("--silero" in args) {
|
||||
check(SileroProcess.isInstalled()) { "Install the Silero desktop runtime first." }
|
||||
SileroProcess().use { engine ->
|
||||
for ((label, text) in listOf(
|
||||
"library" to "На двери висел старый замок, а вдали виднелся замок на холме.",
|
||||
"yo" to "Ежик шел по лесу. Ребенок пьет теплое молоко.",
|
||||
)) {
|
||||
val audio = engine.synthesize(text, "xenia")
|
||||
check(audio.samples.any { kotlin.math.abs(it) > 0.001f })
|
||||
val file = File(output, "silero-desktop-$label.wav")
|
||||
audio.writeWav(file)
|
||||
println("Silero desktop worker: ${file.absolutePath}; ${audio.samples.size.toDouble() / audio.sampleRate}s")
|
||||
if ("--play" in args) playDesktopAudio(audio)
|
||||
}
|
||||
}
|
||||
return@runBlocking
|
||||
}
|
||||
check(SupertonicTts.isInstalled()) { "Run python3 scripts/download-tts-model.py first." }
|
||||
lateinit var engine: SupertonicTts
|
||||
val loadTime = measureNanoTime { engine = SupertonicTts() } / 1e9
|
||||
println("Supertonic 3; CPU only; up to 4 threads; 8 steps; model load: %.3f s".format(Locale.ROOT, loadTime))
|
||||
val stressTrial = "--stress" in args
|
||||
val ruaccentTrial = "--ruaccent" in args
|
||||
val russianText = "В библиотеке было тихо. Она открыла книгу и начала читать. За окном шёл осенний дождь. На двери висел старый замок, а вдали виднелся замок на холме. Завтра, подумала она, мы наконец доберёмся до моря."
|
||||
val accentor = if (ruaccentTrial) RuAccentProcess() else null
|
||||
val samples = if (ruaccentTrial) linkedMapOf("ru-plain" to russianText, "ru-ruaccent" to russianText) else if (stressTrial) linkedMapOf(
|
||||
"ru-plain" to "Перед нами старый замок. Они стояли около замка.",
|
||||
"ru-castle" to "П+еред н+ами ст+арый з+амок. Он+и сто+яли +около з+амка.",
|
||||
"ru-lock" to "П+еред н+ами ст+арый зам+ок. Он+и сто+яли +около замк+а.",
|
||||
) else linkedMapOf(
|
||||
"en" to "The library was quiet. She opened the book and began to read. Outside, the autumn rain tapped softly against the window. Tomorrow, she thought, they would finally reach the coast.",
|
||||
"ru" to russianText,
|
||||
)
|
||||
engine.use {
|
||||
try {
|
||||
for (voice in if (stressTrial || ruaccentTrial) listOf("F1") else listOf("F1", "M1")) for ((label, text) in samples) {
|
||||
val language = if (stressTrial || ruaccentTrial) "ru" else label
|
||||
val accentStart = System.nanoTime()
|
||||
val accented = if (label == "ru-ruaccent") accentor!!.accentuate(text) else text
|
||||
if (label == "ru-ruaccent") println("RuAccent (including first load): %.3f s".format(Locale.ROOT, (System.nanoTime() - accentStart) / 1e9))
|
||||
val spoken = SupertonicTts.russianStressToUnicode(accented)
|
||||
lateinit var audio: SpeechAudio
|
||||
val elapsed = measureNanoTime {
|
||||
audio = joinSpeechAudio(SupertonicTts.chunks(spoken).map { chunk ->
|
||||
SpeechAudio(engine.synthesize(chunk, voice, language, seed = 42), engine.sampleRate)
|
||||
})
|
||||
} / 1e9
|
||||
val duration = audio.samples.size.toDouble() / audio.sampleRate
|
||||
check(audio.samples.all { it.isFinite() } && audio.samples.any { kotlin.math.abs(it) > 0.001f }) {
|
||||
"Invalid or silent model output"
|
||||
}
|
||||
val file = File(output, "$label-$voice.wav")
|
||||
audio.writeWav(file)
|
||||
File(output, "$label-$voice.txt").writeText(spoken + "\n")
|
||||
println("$label/$voice: %.3f s synthesis, %.3f s audio, RTF %.3f; %s".format(
|
||||
Locale.ROOT, elapsed, duration, elapsed / duration, file.absolutePath,
|
||||
))
|
||||
if ("--play" in args) playDesktopAudio(audio)
|
||||
}
|
||||
} finally {
|
||||
accentor?.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Supertone Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Local stdin/stdout JSON protocol. Book text never leaves this process tree."""
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HUB_DISABLE_TELEMETRY="1", TOKENIZERS_PARALLELISM="false")
|
||||
|
||||
|
||||
def load(model):
|
||||
# Bound ORT worker counts before RUAccent imports its session constructor.
|
||||
import onnxruntime as ort
|
||||
original = ort.InferenceSession
|
||||
|
||||
def cpu_session(path, *args, **kwargs):
|
||||
options = ort.SessionOptions()
|
||||
options.intra_op_num_threads = 4
|
||||
options.inter_op_num_threads = 1
|
||||
kwargs["sess_options"] = options
|
||||
kwargs["providers"] = ["CPUExecutionProvider"]
|
||||
return original(path, *args, **kwargs)
|
||||
|
||||
ort.InferenceSession = cpu_session
|
||||
from ruaccent import RUAccent
|
||||
accentor = RUAccent()
|
||||
accentor.load(omograph_model_size="turbo3.1", use_dictionary=True, device="CPU", workdir=str(model), tiny_mode=False)
|
||||
return accentor
|
||||
|
||||
|
||||
def main():
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
accentor = load(Path(sys.argv[1]))
|
||||
if "--check" in sys.argv:
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
result = accentor.process_all("На двери висит замок.")
|
||||
print(result, flush=True)
|
||||
return
|
||||
print(json.dumps({"ready": True}), flush=True)
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
text = json.loads(line)["text"]
|
||||
if not isinstance(text, str) or len(text) > 2000:
|
||||
raise ValueError("Expected text of at most 2000 characters")
|
||||
start = time.monotonic()
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
accented = accentor.process_all(text)
|
||||
print(json.dumps({"text": accented, "seconds": time.monotonic() - start}, ensure_ascii=False), flush=True)
|
||||
except Exception as error:
|
||||
# Do not echo private input or exception text into application logs.
|
||||
print(json.dumps({"error": type(error).__name__}), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Persistent CPU-only Silero worker; local JSON lines, float32 audio in base64."""
|
||||
import array
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
||||
os.environ["TORCH_HOME"] = str(Path(sys.argv[1]).parent / "torch")
|
||||
|
||||
|
||||
def main():
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
import torch
|
||||
torch.set_num_threads(4)
|
||||
torch.set_num_interop_threads(1)
|
||||
model = torch.package.PackageImporter(str(Path(sys.argv[1]) / "model.pt")).load_pickle("tts_models", "model")
|
||||
model.to(torch.device("cpu"))
|
||||
print(json.dumps({"ready": True}), flush=True)
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
request = json.loads(line)
|
||||
text = request["text"]
|
||||
voice = request["voice"]
|
||||
if not isinstance(text, str) or not 0 < len(text) <= 2000 or voice not in model.speakers:
|
||||
raise ValueError("Invalid speech request")
|
||||
# Preserve explicit reader/book stress in Silero's native notation.
|
||||
text = re.sub(r"([аеёиоуыэюяАЕЁИОУЫЭЮЯ])\u0301", r"+\1", text)
|
||||
with contextlib.redirect_stdout(sys.stderr), torch.inference_mode():
|
||||
audio = model.apply_tts(text=text, speaker=voice, sample_rate=48000,
|
||||
put_accent=True, put_yo=True,
|
||||
put_stress_homo=True, put_yo_homo=True).detach().cpu().flatten()
|
||||
if not audio.numel() or not torch.isfinite(audio).all():
|
||||
raise ValueError("Invalid model audio")
|
||||
samples = array.array("f", audio.tolist())
|
||||
if sys.byteorder != "little":
|
||||
samples.byteswap()
|
||||
print(json.dumps({"sampleRate": 48000, "audio": base64.b64encode(samples.tobytes()).decode("ascii")}), flush=True)
|
||||
except Exception as error:
|
||||
print(json.dumps({"error": type(error).__name__}), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,186 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.awaitCancellation
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.test.advanceUntilIdle
|
||||
import kotlinx.coroutines.test.runCurrent
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import net.sergeych.toread.ReadAloudSentence
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNotNull
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
class DesktopReadAloudPlayerTest {
|
||||
private fun sentence(index: Int, text: String) = ReadAloudSentence(index, index, 0, text.length, text)
|
||||
private fun audio(id: Int) = SpeechAudio(floatArrayOf(id.toFloat()), 24000)
|
||||
|
||||
@Test
|
||||
fun sileroDefaultsToXeniaRoutesEnglishAndResetsVoiceOnEngineChange() = runTest {
|
||||
val calls = mutableListOf<String>()
|
||||
var accented = false
|
||||
val player = DesktopReadAloudPlayer(this, { text, voice -> calls += "super:$voice:$text"; audio(1) },
|
||||
{}, { true }, { accented = true; it }, { text, voice -> calls += "silero:$voice:$text"; audio(1) })
|
||||
assertEquals("silero", player.settingsState.value.selectedEngineId)
|
||||
assertEquals("xenia", player.settingsState.value.selectedVoiceId)
|
||||
player.prepare(listOf(sentence(0, "Привет."), sentence(1, "Hello."), sentence(2, "Привет, John.")), 0)
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertEquals(listOf("silero:xenia:Привет.", "super:F1:Hello.", "super:F1:Привет, John."), calls)
|
||||
assertFalse(accented)
|
||||
player.selectVoice("F1")
|
||||
assertEquals("xenia", player.settingsState.value.selectedVoiceId)
|
||||
player.selectEngine("supertonic-3")
|
||||
assertEquals("F1", player.settingsState.value.selectedVoiceId)
|
||||
player.selectEngine("silero")
|
||||
assertEquals("xenia", player.settingsState.value.selectedVoiceId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun switchingAwayFromSileroCancelsPendingSpeech() = runTest {
|
||||
var cancelled = false
|
||||
var played = 0
|
||||
val player = DesktopReadAloudPlayer(this, { _, _ -> audio(1) }, { played++ }, { true },
|
||||
silero = { _, _ -> try { awaitCancellation() } finally { cancelled = true } })
|
||||
player.prepare(listOf(sentence(0, "Привет.")), 0)
|
||||
player.play()
|
||||
runCurrent()
|
||||
player.selectEngine("supertonic-3")
|
||||
advanceUntilIdle()
|
||||
assertTrue(cancelled)
|
||||
assertEquals(1, played)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun stopDuringSynthesisDiscardsAudioAndDoesNotReportAnError() = runTest {
|
||||
var started = false
|
||||
var played = false
|
||||
val player = DesktopReadAloudPlayer(this, { _, _ -> started = true; awaitCancellation() }, { played = true }, { true })
|
||||
player.prepare(listOf(sentence(7, "First.")), 0)
|
||||
player.play()
|
||||
runCurrent()
|
||||
assertTrue(started)
|
||||
assertTrue(player.state.value.buffering)
|
||||
player.stop()
|
||||
runCurrent()
|
||||
assertFalse(played)
|
||||
assertFalse(player.state.value.active)
|
||||
assertNull(player.settingsState.value.message)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun skipCancelsOldSynthesisAndStartsRequestedSentence() = runTest {
|
||||
val played = mutableListOf<Int>()
|
||||
val player = DesktopReadAloudPlayer(this, { text, _ ->
|
||||
if (text == "First.") awaitCancellation() else audio(2)
|
||||
}, { played += it.samples[0].toInt(); awaitCancellation() }, { true })
|
||||
player.prepare(listOf(sentence(10, "First."), sentence(20, "Second.")), 0)
|
||||
player.play()
|
||||
runCurrent()
|
||||
player.skip(1)
|
||||
runCurrent()
|
||||
assertEquals(listOf(2), played)
|
||||
assertEquals(20, player.state.value.sentenceIndex)
|
||||
player.stop()
|
||||
runCurrent()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun voiceChangeCancelsPlaybackAndRestartsCurrentSentence() = runTest {
|
||||
val voices = mutableListOf<String>()
|
||||
var cancelledAudio = 0
|
||||
val player = DesktopReadAloudPlayer(this, { _, voice -> voices += voice; audio(1) }, {
|
||||
try { awaitCancellation() } finally { cancelledAudio++ }
|
||||
}, { true })
|
||||
player.prepare(listOf(sentence(3, "Hello.")), 0)
|
||||
player.play()
|
||||
runCurrent()
|
||||
player.selectVoice("M1")
|
||||
runCurrent()
|
||||
assertEquals(listOf("F1", "M1"), voices)
|
||||
assertEquals(1, cancelledAudio)
|
||||
assertEquals(3, player.state.value.sentenceIndex)
|
||||
player.stop()
|
||||
runCurrent()
|
||||
assertEquals(2, cancelledAudio)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lookaheadDoesNotAdvanceHighlightBeforePlayback() = runTest {
|
||||
val synthesized = mutableListOf<String>()
|
||||
val player = DesktopReadAloudPlayer(this, { text, _ -> synthesized += text; audio(1) }, {
|
||||
awaitCancellation()
|
||||
}, { true })
|
||||
player.prepare(listOf(sentence(4, "First."), sentence(8, "Second."), sentence(12, "Third.")), 0)
|
||||
player.play()
|
||||
runCurrent()
|
||||
assertEquals(listOf("First.", "Second."), synthesized)
|
||||
assertEquals(4, player.state.value.sentenceIndex)
|
||||
player.stop()
|
||||
runCurrent()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun fullQueueUsesSpokenTextAndFinishes() = runTest {
|
||||
val synthesized = mutableListOf<String>()
|
||||
val player = DesktopReadAloudPlayer(this, { text, _ -> synthesized += text; audio(1) }, { delay(10) }, { true })
|
||||
player.prepare(listOf(sentence(0, "1").copy(spokenText = "One.", pauseBeforeMillis = 20), sentence(1, "Two.")), 0)
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertEquals(listOf("One.", "Two."), synthesized)
|
||||
assertFalse(player.state.value.playing)
|
||||
assertFalse(player.state.value.active)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun accentStageCanBeComparedWithPlainEngineAndNeverChangesOriginalText() = runTest {
|
||||
val inputs = mutableListOf<String>()
|
||||
val original = sentence(0, "замок")
|
||||
val player = DesktopReadAloudPlayer(this, { text, _ -> inputs += text; audio(1) }, {}, { true }, { "з+амок" })
|
||||
player.prepare(listOf(original), 0)
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertEquals(listOf("за\u0301мок"), inputs)
|
||||
assertEquals("замок", original.spokenText)
|
||||
player.selectEngine("supertonic-3")
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertEquals(listOf("за\u0301мок", "замок"), inputs)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun accentedTextIsRechunkedToFitTtsWithoutDiscardingMarks() = runTest {
|
||||
val inputs = mutableListOf<String>()
|
||||
val marked = "а\u0301 ".repeat(110).trim()
|
||||
val player = DesktopReadAloudPlayer(this, { text, _ -> inputs += text; audio(1) }, {}, { true }, { marked })
|
||||
player.prepare(listOf(sentence(0, "а ".repeat(110))), 0)
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertTrue(inputs.size > 1)
|
||||
assertTrue(inputs.all { it.length <= 300 })
|
||||
assertEquals(marked.replace(" ", ""), inputs.joinToString("").replace(" ", ""))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun missingModelAndInferenceErrorsRemainVisibleAndRetryable() = runTest {
|
||||
var installed = false
|
||||
var calls = 0
|
||||
val player = DesktopReadAloudPlayer(this, { _, _ -> calls++; error("bad model") }, {}, { installed })
|
||||
player.prepare(listOf(sentence(0, "Hello.")), 0)
|
||||
player.play()
|
||||
assertEquals(0, calls)
|
||||
assertFalse(player.state.value.playing)
|
||||
assertNotNull(player.settingsState.value.message)
|
||||
installed = true
|
||||
player.play()
|
||||
advanceUntilIdle()
|
||||
assertEquals(1, calls)
|
||||
assertTrue(player.state.value.active)
|
||||
assertFalse(player.state.value.buffering)
|
||||
assertNotNull(player.settingsState.value.message)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package net.sergeych.toread.tts
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class SupertonicTextTest {
|
||||
@Test
|
||||
fun longSentencesAndUnbrokenWordsAreBoundedWithoutLosingText() {
|
||||
for (text in listOf("Слово ".repeat(150), "a".repeat(601), "a".repeat(299) + "😀" + "b".repeat(301), "а".repeat(300) + "\u0301" + "а".repeat(50))) {
|
||||
val chunks = SupertonicTts.chunks(text)
|
||||
assertTrue(chunks.all { it.length <= 300 && it.isNotBlank() })
|
||||
assertEquals(text.filterNot(Char::isWhitespace), chunks.joinToString("").filterNot(Char::isWhitespace))
|
||||
assertTrue(chunks.none { Character.isHighSurrogate(it.last()) || Character.isLowSurrogate(it.first()) })
|
||||
assertTrue(chunks.none { it.first() == '\u0301' })
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun languageSelectionHandlesRussianEnglishAndMixedText() {
|
||||
assertEquals("ru", SupertonicTts.languageOf("Ёж шёл домой."))
|
||||
assertEquals("en", SupertonicTts.languageOf("The library."))
|
||||
assertEquals("na", SupertonicTts.languageOf("Он сказал Hello."))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preprocessingDoesNotInsertPredictedStress() {
|
||||
assertEquals("<ru>На двери висит замок.</ru>", SupertonicTts.prepareText("На двери висит замок", "ru"))
|
||||
assertEquals("<en>Hello!</en>", SupertonicTts.prepareText("Hello !", "en"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun ruaccentStressBecomesCombiningAcuteOnCyrillicVowels() {
|
||||
assertEquals("за\u0301мок замо\u0301к Ё\u0301лка", SupertonicTts.russianStressToUnicode("з+амок зам+ок +Ёлка"))
|
||||
assertEquals("2+2 +a +б а\u0301", SupertonicTts.russianStressToUnicode("2+2 +a +б а\u0301"))
|
||||
assertEquals("<ru>За\u0301мок.</ru>", SupertonicTts.prepareText("З+амок", "ru"))
|
||||
assertEquals("<ru>За\u0301мок.</ru>", SupertonicTts.prepareText("За\u0301мок", "ru"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pcmConversionClipsAndHandlesNonFiniteSamples() {
|
||||
val bytes = SpeechAudio(floatArrayOf(-2f, 2f, Float.NaN), 44100).pcm16()
|
||||
assertEquals(listOf(1, 128, 255, 127, 0, 0), bytes.map { it.toInt() and 255 })
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user