diff --git a/.claude/skills/adding-normalizer/SKILL.md b/.claude/skills/adding-normalizer/SKILL.md index d981c4a0..f353340e 100644 --- a/.claude/skills/adding-normalizer/SKILL.md +++ b/.claude/skills/adding-normalizer/SKILL.md @@ -7,7 +7,7 @@ Read [AGENTS.md](../../../AGENTS.md) and [CONTRIBUTING.md](../../../CONTRIBUTING # Adding a language normalizer -Text normalization (numbers, money, time, URLs, etc. spelled out before phonemization) is per language. Only English is implemented; other languages skip normalization and rely on the phonemizer. +Text normalization (numbers, money, time, URLs, etc. spelled out before phonemization) is per language. English and Brazilian Portuguese are implemented; other languages skip normalization and rely on the phonemizer. ## Where things live @@ -16,15 +16,27 @@ Text normalization (numbers, money, time, URLs, etc. spelled out before phonemiz - `base.py`: `Normalizer`, the contract. `pass_order()` is the pass order, `normalize()` runs it. The language-neutral passes (quotes, CJK punctuation, whitespace) are implemented; every other pass is a hook that returns its input unchanged until overridden. Override `pass_order()` if a language needs a different order. - `english.py`: `EnglishNormalizer`, the reference implementation. - `english_data.py`: the English tables, TLDs, units, symbol readings, currencies. +- `portuguese.py`: `PortugueseNormalizer`, a small second language (numbers, currency, times, units). The closer template for a new one. - `__init__.py`: `LANGUAGES`, the registry. Lang codes come from `lang_codes` on each class. ## Adding one 1. Create `normalization/.py` with a `Normalizer` subclass. Set `lang_codes` to the Kokoro pipeline codes it serves (`p` for Portuguese, `e` for Spanish, etc.). Override the hooks you need, `numbers` first (money is spelled out there too); espeak already reads bare digits, so the value is in currency, time, and units. `num2words` is in the lock and covers cardinals for most languages. 2. Add an instance to `LANGUAGES` in `__init__.py`. -3. Add `api/tests/test_normalizer_.py` with a `CASES` table of `(input, expected)` rows, mirroring `test_normalizer.py`. `test_normalizer_contract.py` runs the never-raises and idempotence properties on every registered language automatically. +3. Add `api/tests/test_normalizer_.py` with a `CASES` table of `(input, expected, options)` rows, mirroring `test_normalizer_portuguese.py`. `test_normalizer_contract.py` runs the never-raises and idempotence properties on every registered language automatically. + +## Gotchas + +- Passes run in order (`units` only with `unit_normalization`, then `times`, then `numbers`), and an early pass hides text from the later ones. Each pattern that takes digits owns its sign and its left boundary (`(?` (OpenAI), `xi-api-key` (ElevenLabs), `X-API-Key` (Cartesia). `/health` and the web player's files stay open. Off by default. Convenience feature, be sure of your security requirements (e.g. if public, put a TLS reverse proxy in front) (#373) - ElevenLabs and Cartesia compatible websocket TTS, for voice agent frameworks (Pipecat, LiveKit Agents etc): - Streamed text input, PCM or MP3 output with word timestamps diff --git a/api/src/services/text_processing/normalization/portuguese.py b/api/src/services/text_processing/normalization/portuguese.py index 43a1f909..badb4da1 100644 --- a/api/src/services/text_processing/normalization/portuguese.py +++ b/api/src/services/text_processing/normalization/portuguese.py @@ -1,7 +1,7 @@ """Brazilian Portuguese text normalization.""" import re -from decimal import Decimal, InvalidOperation +from decimal import Decimal from num2words import num2words @@ -18,14 +18,24 @@ class PortugueseNormalizer(Normalizer): r"(?P,\d{1,18})?(?![\w:])" ) CURRENCY_PATTERN = re.compile( - r"(?PR\$|US\$|[$€£])\s*" - r"(?P\d{1,3}(?:\.\d{3}){1,9}(?:,\d{1,2})?|\d{1,30}(?:,\d{1,2})?)", + r"(?P(?R\$|US\$|[$€£])\s*" + r"(?P\d{1,3}(?:\.\d{3}){1,9}(?:,\d{1,2})?" + r"|\d{1,30}(?:,\d{1,2}|(?P\.\d{2}))?)" + r"(?!\d|[.,]\d)", re.IGNORECASE, ) - TIME_PATTERN = re.compile(r"(?\d{1,2}):(?P\d{2})(?![\d:])") - UNIT_PATTERN = re.compile( + PERCENT_PATTERN = re.compile(r"(?<=\d)\s?%") + TIME_PATTERN = re.compile( + r"(?\d{1,2})(?P[:h])(?P\d{2})(?:\s?(?:h|min|m)\b)?(?![\w:])", + re.IGNORECASE, + ) + QUANTITY = ( + r"(?P(?(?:\d{1,3}(?:\.\d{3}){1,9}|\d{1,30})(?:,\d{1,18})?)\s*" - r"(?Pkm/h|m/s|km|cm|mm|kg|mg|g|ml|l|h|min|s|°c|°f)\b", + ) + HOUR_PATTERN = re.compile(QUANTITY + r"(?Ph)\b") + UNIT_PATTERN = re.compile( + QUANTITY + r"(?Pkm/h|m/s|km|cm|mm|kg|mg|g|ml|l|min|s|°c|°f)\b", re.IGNORECASE, ) UNITS = { @@ -45,24 +55,30 @@ class PortugueseNormalizer(Normalizer): "°c": ("grau Celsius", "graus Celsius"), "°f": ("grau Fahrenheit", "graus Fahrenheit"), } + FEMININE = { + "um": "uma", + "dois": "duas", + "duzentos": "duzentas", + "trezentos": "trezentas", + "quatrocentos": "quatrocentas", + "quinhentos": "quinhentas", + "seiscentos": "seiscentas", + "setecentos": "setecentas", + "oitocentos": "oitocentas", + "novecentos": "novecentas", + } def units(self, text: str) -> str: - def spell(match: re.Match[str]) -> str: - amount = self._parse_amount(match.group("number")) - unit = self.UNITS[match.group("unit").lower()] - name = unit[0] if amount == 1 else unit[1] - return f"{match.group('number')} {name}" - - return self.UNIT_PATTERN.sub(spell, text) + return self.UNIT_PATTERN.sub(self._unit, text) def times(self, text: str) -> str: + text = self.HOUR_PATTERN.sub(self._unit, text) + def spell(match: re.Match[str]) -> str: hour, minute = int(match.group("hour")), int(match.group("minute")) - if hour > 23 or minute > 59: + if minute > 59 or (hour > 23 and match.group("sep") == ":"): return match.group() - hour_words = num2words(hour, lang="pt_BR") - if hour in (1, 21): - hour_words = hour_words.removesuffix("um") + "uma" + hour_words = self._feminine(num2words(hour, lang="pt_BR")) parts = [hour_words, "hora" if hour == 1 else "horas"] if minute: parts.extend( @@ -78,23 +94,42 @@ def spell(match: re.Match[str]) -> str: def numbers(self, text: str) -> str: text = self.CURRENCY_PATTERN.sub(self._currency, text) + text = self.PERCENT_PATTERN.sub(" por cento", text) def spell(match: re.Match[str]) -> str: - raw = match.group("whole").replace(".", "") - if match.group("fraction"): - raw += "." + match.group("fraction")[1:] + whole = int(match.group("whole").replace(".", "")) try: - value = Decimal(raw) - if match.group("sign") == "-": - value = -value - return num2words(value, lang="pt_BR") - except (InvalidOperation, NotImplementedError, OverflowError, ValueError): + parts = [num2words(whole, lang="pt_BR")] + except (NotImplementedError, OverflowError, ValueError): return match.group() + if match.group("fraction"): + digits = match.group("fraction")[1:].rstrip("0") or "0" + parts.append("vírgula") + parts.extend(num2words(int(digit), lang="pt_BR") for digit in digits) + if match.group("sign") == "-": + parts.insert(0, "menos") + return " ".join(parts) return self.NUMBER_PATTERN.sub(spell, text) + def _unit(self, match: re.Match[str]) -> str: + amount = self._parse_amount(match.group("number")) + unit = self.UNITS[match.group("unit").lower()] + name = unit[0] if amount == 1 else unit[1] + if unit[0] == "hora" and amount == int(amount): + try: + words = self._feminine(num2words(int(amount), lang="pt_BR")) + sign = "menos " if match.group("sign") == "-" else "" + return f"{sign}{words} {name}" + except (NotImplementedError, OverflowError, ValueError): + pass + return f"{match.group('sign') or ''}{match.group('number')} {name}" + def _currency(self, match: re.Match[str]) -> str: - amount = self._parse_amount(match.group("amount")) + raw = match.group("amount") + if match.group("dot_cents"): + raw = raw.replace(".", ",") + amount = self._parse_amount(raw) symbol = match.group("symbol").upper() currency = { "R$": ("real", "reais", "centavo", "centavos"), @@ -109,8 +144,12 @@ def _currency(self, match: re.Match[str]) -> str: integer_part = int(amount) cents = int((amount - integer_part) * 100) try: - parts = [num2words(integer_part, lang="pt_BR")] - parts.append(currency[0] if integer_part == 1 else currency[1]) + words = num2words(integer_part, lang="pt_BR") + if currency[0] == "libra": + words = self._feminine(words) + if words.endswith(("ão", "ões")): + words += " de" + parts = [words, currency[0] if integer_part == 1 else currency[1]] if cents: parts.extend( ( @@ -119,10 +158,25 @@ def _currency(self, match: re.Match[str]) -> str: currency[2] if cents == 1 else currency[3], ) ) + if match.group("sign") == "-": + parts.insert(0, "menos") return " ".join(parts) except (NotImplementedError, OverflowError, ValueError): return match.group() + def _feminine(self, words: str) -> str: + tokens = words.split() + start = max( + ( + i + 1 + for i, token in enumerate(tokens) + if token.rstrip(",").endswith(("ão", "ões")) + ), + default=0, + ) + tokens[start:] = [self.FEMININE.get(token, token) for token in tokens[start:]] + return " ".join(tokens) + @staticmethod def _parse_amount(raw: str) -> Decimal: return Decimal(raw.replace(".", "").replace(",", ".")) diff --git a/api/tests/test_normalizer_portuguese.py b/api/tests/test_normalizer_portuguese.py index 7fed97e7..099e96ad 100644 --- a/api/tests/test_normalizer_portuguese.py +++ b/api/tests/test_normalizer_portuguese.py @@ -21,6 +21,50 @@ ("Às 9:00", "Às nove horas", OPTIONS), ("Às 1:00", "Às uma hora", OPTIONS), ("São 21:00", "São vinte e uma horas", OPTIONS), + ("São 2:00", "São duas horas", OPTIONS), + ("São 22:30", "São vinte e duas horas e trinta minutos", OPTIONS), + ("Durou 1 h", "Durou uma hora", OPTIONS), + ("Durou 2 h", "Durou duas horas", UNIT_OPTIONS), + ("Custa £2", "Custa duas libras", OPTIONS), + ("Custa £200", "Custa duzentas libras", OPTIONS), + ("Custa £2.000.000", "Custa dois milhões de libras", OPTIONS), + ( + "Custa £2.234.201", + "Custa dois milhões, duzentas e trinta e quatro mil, duzentas e uma libras", + OPTIONS, + ), + ("Custa R$ 1.000.000,00", "Custa um milhão de reais", OPTIONS), + ("Custa R$ 2.500.000", "Custa dois milhões e quinhentos mil reais", OPTIONS), + ("Custa R$ 5, mas", "Custa cinco reais, mas", OPTIONS), + ("Custa $1.50", "Custa um dólar e cinquenta centavos", OPTIONS), + ("Custa R$ 1.500", "Custa mil e quinhentos reais", OPTIONS), + ("Custa R$ 12,505", "Custa R$ doze vírgula cinco zero cinco", OPTIONS), + ("Saldo -R$ 1,50", "Saldo menos um real e cinquenta centavos", OPTIONS), + ("Subiu 50%", "Subiu cinquenta por cento", OPTIONS), + ("Caiu -0,5", "Caiu menos zero vírgula cinco", OPTIONS), + ( + "Vale 1,000000000000000001", + "Vale um vírgula " + "zero " * 17 + "um", + OPTIONS, + ), + ("Às 12:30h", "Às doze horas e trinta minutos", OPTIONS), + ("Às 12:30 h", "Às doze horas e trinta minutos", OPTIONS), + ("Às 14:30H", "Às catorze horas e trinta minutos", OPTIONS), + ("Às 12:30abc", "Às 12:30abc", OPTIONS), + ("Modelo abc2h", "Modelo abc2h", OPTIONS), + ("Às 12h30", "Às doze horas e trinta minutos", OPTIONS), + ("Dura 2h30min", "Dura duas horas e trinta minutos", UNIT_OPTIONS), + ("Modelo abc2h30", "Modelo abc2h30", OPTIONS), + ("Às 14h", "Às catorze horas", OPTIONS), + ("Durou 30h45", "Durou trinta horas e quarenta e cinco minutos", OPTIONS), + ("Aberto 24h", "Aberto vinte e quatro horas", OPTIONS), + ("Lápis 2H", "Lápis 2H", UNIT_OPTIONS), + ("Atraso de -1 h", "Atraso de menos uma hora", OPTIONS), + ("Adiantou +1 h", "Adiantou uma hora", OPTIONS), + ("Saldo +R$ 5", "Saldo cinco reais", OPTIONS), + ("Desceu -2 km", "Desceu menos dois quilômetros", UNIT_OPTIONS), + ("Das 10-2 h", "Das dez-duas horas", OPTIONS), + ("Durou 1000000000000000000 h", "Durou 1000000000000000000 horas", OPTIONS), ("Timestamp: 12:34:56", "Timestamp: 12:34:56", OPTIONS), ("São 24:75", "São 24:75", OPTIONS), ("Fica a 2 km", "Fica a dois km", OPTIONS), diff --git a/api/tests/test_normalizer_redos.py b/api/tests/test_normalizer_redos.py index 1054d310..e181c42f 100644 --- a/api/tests/test_normalizer_redos.py +++ b/api/tests/test_normalizer_redos.py @@ -77,10 +77,11 @@ def test_number_floods_scale_linearly(): def test_portuguese_number_floods_scale_linearly(): - """Portuguese number and currency regexes must stay linear on request text.""" - for unit in ("9", "1.000", "R$1,"): - small = _elapsed_portuguese(unit * 2_000) - large = _elapsed_portuguese(unit * 8_000) + """Portuguese number, currency, time and unit regexes must stay linear on request text.""" + opts = NormalizationOptions(unit_normalization=True) + for unit in ("9", "1.000", "R$1,", "1 h", "12:30h", "12h30", "1,5 km/h ", "1%"): + small = _elapsed_portuguese(unit * 2_000, opts) + large = _elapsed_portuguese(unit * 8_000, opts) assert large / max(small, 0.05) < 9.0, unit