[Python] How can I ensure that Azure Text-to-Speech properly pronounces word-for-word...

I'm working on an app using Azure, Gemini, Python, and Dart, and I want to make sure the pronunciation between languages is spot on. For example, I want to translate between German and Spanish: the goal is for 'hallo' -> 'hola' to be pronounced correctly in both languages. The same goes for English and Spanish 'hello' -> 'hola'. Azure does well with sentences, but struggles with word-for-word translations.

Here's my code:

• translation_service.py

class TranslationService:
def __init__(self):
load_dotenv()
api_key = os.getenv("GEMINI_API_KEY")
if not api_key:
raise ValueError("GEMINI_API_KEY not found in environment variables")

genai.configure(api_key=api_key)

self.generation_config = {
"temperature": 1,
"top_p": 0.95,
"top_k": 40,
"max_output_tokens": 8192,
"response_mime_type": "text/plain",
}

self.model = GenerativeModel(
model_name="gemini-2.0-flash-exp",
generation_config=self.generation_config
)

self.tts_service = EnhancedTTSService()

# Initialize chat session with translation instructions
self.chat_session = self.model.start_chat(
history=[
{
"role": "user",
"parts": [
"""

Text
"
(Could be any phrase or word)
"

German Translation:
Conversational-native:
"Ich suche einen Job, damit ich finanziell unabhängig sein kann."
word by word Conversational-native German-Spanish:
"Ich (Yo) suche (busco) einen (un) Job (trabajo), damit (para que) ich (yo) finanziell (económicamente) unabhängig (independiente) sein (ser) kann (pueda)."


English Translation:

Conversational-native:
"I'm looking for a job so I can be financially independent."
word by word Conversational-native English-Spanish:
"I'm (Yo estoy) looking for (buscando) a job (un trabajo) so (para que) I (yo) can be (pueda ser) financially (económicamente) independent (independiente)."



"""
]
}
]
)

def _restore_accents(self, text: str) -> str:
"""Restore proper accents and special characters."""
accent_map = {
"a": "á", "e": "é", "i": "í", "o": "ó", "u": "ú", "n": "ñ",
"A": "Á", "E": "É", "I": "Í", "O": "Ó", "U": "Ú", "N": "Ñ",
}

patterns = {
r"([aeiou])´": lambda m: accent_map[m.group(1)],
r"([AEIOU])´": lambda m: accent_map[m.group(1)],
r"n~": "ñ",
r"N~": "Ñ",
}

for pattern, replacement in patterns.items():
if callable(replacement):
text = re.sub(pattern, replacement, text)
else:
text = re.sub(pattern, replacement, text)

return text

async def process_prompt(self, text: str, source_lang: str, target_lang: str) -> Translation:
try:
response = self.chat_session.send_message(text)
generated_text = response.text

print(f"Generated text from Gemini: {generated_text[:100]}...")

audio_filename = await self.tts_service.text_to_speech(
text=generated_text
)

if audio_filename:
print(f"Successfully generated audio: {audio_filename}")
else:
print("Audio generation failed")

return Translation(
original_text=text,
translated_text=generated_text,
source_language=source_lang,
target_language=target_lang,
audio_path=audio_filename,
translations={"main": generated_text},
word_by_word=self._generate_word_by_word(text, generated_text),
grammar_explanations=self._generate_grammar_explanations(generated_text)
)

except Exception as e:
print(f"Error in process_prompt: {str(e)}")
raise Exception(f"Translation processing failed: {str(e)}")


def _generate_word_by_word(self, original: str, translated: str) -> dict[str, dict[str, str]]:
"""Generate word-by-word translation mapping."""
result = {}
original_words = original.split()
translated_words = translated.split()

for i, word in enumerate(original_words):
if i < len(translated_words):
result[word] = {
"translation": translated_words,
"pos": "unknown",
}
return result


def _auto_fix_spelling(self, text: str) -> str:
"""Fix spelling in the given text."""
words = re.findall(r"\b\w+\b|[^\w\s]", text)
corrected_words = []

for word in words:
if not re.match(r"\w+", word):
corrected_words.append(word)
continue

if self.spell.unknown([word]):
correction = self.spell.correction(word)
if correction:
if word.isupper():
correction = correction.upper()
elif word[0].isupper():
correction = correction.capitalize()
word = correction

corrected_words.append(word)

return " ".join(corrected_words)

• tts_service.py

from azure.cognitiveservices.speech.audio import AudioOutputConfig
import os
from typing import Optional
from datetime import datetime
import asyncio
import re

class EnhancedTTSService:
def __init__(self):
# Initialize Speech Config
self.subscription_key = os.getenv("AZURE_SPEECH_KEY")
self.region = os.getenv("AZURE_SPEECH_REGION")

if not self.subscription_key or not self.region:
raise ValueError("Azure Speech credentials not found in environment variables")

# Create speech config
self.speech_config = SpeechConfig(
subscription=self.subscription_key,
region=self.region
)
self.speech_config.set_speech_synthesis_output_format(
SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
)

# Voice mapping with specific styles and roles
self.voice_mapping = {
'en': 'en-US-JennyMultilingualNeural',
'es': 'es-ES-ArabellaMultilingualNeural',
'de': 'de-DE-SeraphinaMultilingualNeural'
}



def _get_temp_directory(self) -> str:
"""Create and return the temporary directory path"""
if os.name == 'nt': # Windows
temp_dir = os.path.join(os.environ.get('TEMP', ''), 'tts_audio')
else: # Unix/Linux
temp_dir = '/tmp/tts_audio'
os.makedirs(temp_dir, exist_ok=True)
return temp_dir

def _detect_language(self, text: str) -> str:
"""Detect the primary language of the text"""
# Simple language detection based on character patterns
if re.search(r'[äöüßÄÖÜ]', text):
return 'de'
elif re.search(r'[áéíóúñ¿¡]', text):
return 'es'
return 'en'

def _generate_ssml(self, text: str) -> str:
"""Generate valid SSML with proper escaping and language tags"""
# Clean the text
text = text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

# Detect primary language
primary_lang = self._detect_language(text)
voice_name = self.voice_mapping.get(primary_lang, self.voice_mapping['en'])

ssml = f"""<?xml version='1.0'?>
<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='{primary_lang}'>
<voice name='{voice_name}'>
<prosody rate="0.95" pitch="0%">
{text}
</prosody>
</voice>
</speak>"""
return ssml

async def text_to_speech(self, text: str, output_path: Optional[str] = None) -> Optional[str]:
"""Convert text to speech with robust error handling"""
synthesizer = None
try:
print(f"\nStarting TTS process for text: {text[:100]}...") # First 100 chars

# Generate output path if not provided
if not output_path:
temp_dir = self._get_temp_directory()
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
output_path = os.path.join(temp_dir, f"speech_{timestamp}.mp3")

# Configure audio output
audio_config = AudioOutputConfig(filename=output_path)

# Create synthesizer for this request
synthesizer = SpeechSynthesizer(
speech_config=self.speech_config,
audio_config=audio_config
)

# Generate and validate SSML
ssml = self._generate_ssml(text)
print(f"Generated SSML length: {len(ssml)} characters")

# Perform synthesis
print("Starting speech synthesis...")
result = await asyncio.get_event_loop().run_in_executor(
None,
lambda: synthesizer.speak_ssml_async(ssml).get()
)

# Handle result
if result.reason == ResultReason.SynthesizingAudioCompleted:
print("Speech synthesis completed successfully")
return os.path.basename(output_path)

elif result.reason == ResultReason.Canceled:
print(f"Speech synthesis canceled: {result.cancellation_details.reason}")
print(f"Error details: {result.cancellation_details.error_details}")
return None

return None

except Exception as e:
print(f"Exception in text_to_speech: {str(e)}")
return None

finally:
# Proper cleanup
if synthesizer:
try:
synthesizer.stop_speaking_async()
except:
pass


This is an example of how the correct pronunciation should sound:

German-Spanish (hello example) (this is the desired output with the correct word-for-word pronunciation)

https://jmp.sh/s/8sftiJ01aUreR3LDYRWn

English-Spanish (hello example) (this is the desired output with the correct word-for-word pronunciation)

https://jmp.sh/s/9MM1LqTqGH1CvddGhA1l

Now let’s do a word-for-word translation, where we’ll focus on pronouncing the Spanish "ñ," "h," and "ll" properly.

Here’s the Spanish sentence:

"Jugo de piña para la niña y jugo de mora para la señora porque están en el hospital y afuera está lloviendo."

Translation:

"I got pineapple juice for the girl and blackberry juice for the lady because they’re in the hospital and it’s raining outside."

German-Spanish (this is the desired output with the correct word-for-word pronunciation)

https://jmp.sh/s/aRFlpZc99Dw18Uexi8uS

English-Spanish (this is the desired output with the correct word-for-word pronunciation)

https://jmp.sh/eY9ZhlTi


Currently I have this pronunciation with the same examples

German-Spanish and English-Spanish (hello example) (which is incorrect because the word-for-word pronunciation is not accurate)

https://jmp.sh/iExSVBGk

Let’s go back to the word-for-word breakdown, again emphasizing Spanish pronunciation for the tricky letters:

"ñ" (sounds like “ny” in canyon, e.g., piña, niña) "h" (silent in Spanish, e.g., hospital) "ll" (varies regionally but often sounds like “y” in yes, e.g., lloviendo). So here’s the sentence again:

"Jugo de piña para la niña y jugo de mora para la señora porque están en el hospital y afuera está lloviendo."

Translation:

"I got pineapple juice for the girl and blackberry juice for the lady because they’re in the hospital and it’s raining outside."

German-Spanish and English-Spanish (which is incorrect because the word-for-word pronunciation is not accurate)

https://jmp.sh/PxKHNWjx


This is the service I use with Azure:



I’ve tried the 'langid' library, but it seems like it doesn’t work for me. My goal is to be able to hear the correct pronunciation of the English-Spanish and German-Spanish word pairs during word-for-word translation.

Thank you.

Continue reading...