pablo/vienalatina
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'claude/relaxed-faraday-h4zd09' of https://github.com/pablovolenski/vienalatina

Browse Source
This commit is contained in:
admin 2026-09-16 14:01:34 +00:00
commit d47cd326bc
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scripts/translation/markdown.py
View File

@ -5,9 +5,11 @@ model has no equivalent, so structure is protected here instead: non-prose
blocks pass through untouched, and inline constructs are masked with opaque blocks pass through untouched, and inline constructs are masked with opaque
placeholders whose survival is verified after the round trip. placeholders whose survival is verified after the round trip.
Placeholders use OpenNMT's protected-sequence convention (U+FF5F/U+FF60). Placeholders are word-shaped (``Zq0Xv``) rather than punctuation. Measured on
SentencePiece keeps these atomic; ``{{x}}``, ``<x>`` and ``%s`` get fragmented M2M100 418M: the OpenNMT ``｟0｠`` convention was dropped on every single
by BPE and dropped by the model. occurrence, because SentencePiece fragments punctuation runs and the model then
fails to copy them. A token that looks like an unknown proper noun gets carried
through, since the model has nothing to translate it to.
""" """
from __future__ import annotations from __future__ import annotations
@ -16,7 +18,8 @@ import re
from .provider import SITE_TO_MODEL, Provider from .provider import SITE_TO_MODEL, Provider
OPEN, CLOSE = "｟", "｠" _MASK = "Zq{}Xv"
_MASK_RE = re.compile(r"Zq\s*(\d+)\s*Xv", re.IGNORECASE)
# Community vocabulary that must reach readers unchanged. Not inherited from # Community vocabulary that must reach readers unchanged. Not inherited from
# the WordPress plugin, which had no glossary at all — edit freely. # the WordPress plugin, which had no glossary at all — edit freely.
@ -43,9 +46,6 @@ _INLINE = (
re.compile(r"https?://\S+"), # bare URLs re.compile(r"https?://\S+"), # bare URLs
) )
_PLACEHOLDER = re.compile(re.escape(OPEN) + r"\s*(\d+)\s*" + re.escape(CLOSE))
class PlaceholderError(RuntimeError): class PlaceholderError(RuntimeError):
"""A masked span did not survive translation intact.""" """A masked span did not survive translation intact."""
@ -57,7 +57,7 @@ class _Masker:
def _take(self, match: re.Match) -> str: def _take(self, match: re.Match) -> str:
self.spans.append(match.group(0)) self.spans.append(match.group(0))
return f"{OPEN}{len(self.spans) - 1}{CLOSE}" return _MASK.format(len(self.spans) - 1)
def mask(self, text: str) -> str: def mask(self, text: str) -> str:
for pattern in _INLINE: for pattern in _INLINE:
@ -68,10 +68,10 @@ class _Masker:
return text return text
def restore(self, text: str) -> str: def restore(self, text: str) -> str:
# Models pad and reorder placeholders; normalise spacing before matching. # Models pad, re-case and reorder placeholders; normalise before matching.
text = _PLACEHOLDER.sub(lambda m: f"{OPEN}{m.group(1)}{CLOSE}", text) text = _MASK_RE.sub(lambda m: _MASK.format(m.group(1)), text)
for index, span in enumerate(self.spans): for index, span in enumerate(self.spans):
token = f"{OPEN}{index}{CLOSE}" token = _MASK.format(index)
seen = text.count(token) seen = text.count(token)
if seen != 1: if seen != 1:
raise PlaceholderError( raise PlaceholderError(