From 448535d6ad0a9fc871beb18f422ad1c2a186516c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 31 Jul 2026 17:50:37 +0000 Subject: [PATCH 1/4] Fix Woodpecker image tags: v2 no longer exists on Docker Hub, use v3 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017z2rT1oN7vMS4ggo23n5WG --- infra/woodpecker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/woodpecker/docker-compose.yml b/infra/woodpecker/docker-compose.yml index 2f3f89f..7969282 100644 --- a/infra/woodpecker/docker-compose.yml +++ b/infra/woodpecker/docker-compose.yml @@ -5,7 +5,7 @@ services: server: - image: woodpeckerci/woodpecker-server:v2 + image: woodpeckerci/woodpecker-server:v3 restart: unless-stopped ports: - "127.0.0.1:8000:8000" @@ -22,7 +22,7 @@ services: - ./data:/var/lib/woodpecker agent: - image: woodpeckerci/woodpecker-agent:v2 + image: woodpeckerci/woodpecker-agent:v3 restart: unless-stopped depends_on: - server From 91bfdabc71be3ff13ac06751680636bfb8c1622a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 31 Jul 2026 17:55:53 +0000 Subject: [PATCH 2/4] Document uid-1000 ownership for Woodpecker v3 data volume Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017z2rT1oN7vMS4ggo23n5WG --- docs/server-setup.md | 1 + infra/woodpecker/docker-compose.yml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/server-setup.md b/docs/server-setup.md index ed6211c..361e76e 100644 --- a/docs/server-setup.md +++ b/docs/server-setup.md @@ -163,6 +163,7 @@ cd /srv/woodpecker sudo cp .env.example .env openssl rand -hex 32 # copy the output sudo nano .env # paste Woodpecker OAuth client ID + secret + the random hex +sudo mkdir -p data && sudo chown -R 1000:1000 data # v3 images run as uid 1000 sudo docker compose up -d ``` diff --git a/infra/woodpecker/docker-compose.yml b/infra/woodpecker/docker-compose.yml index 7969282..36dfae9 100644 --- a/infra/woodpecker/docker-compose.yml +++ b/infra/woodpecker/docker-compose.yml @@ -1,6 +1,7 @@ # Woodpecker CI — server + one agent on the same VPS. -# Copy this directory to /srv/woodpecker/, create .env from .env.example, -# then: docker compose up -d +# Copy this directory to /srv/woodpecker/, create .env from .env.example, then: +# sudo mkdir -p data && sudo chown -R 1000:1000 data # v3 images run as uid 1000 +# docker compose up -d # Web UI is bound to localhost only; Caddy proxies ci.vienalatina.com to it. services: From adeecb5e9241e4688db0e7db5bae93d4cef890fa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 14:30:20 +0000 Subject: [PATCH 3/4] Install git in the translate step's container translate.py shells out to git (rev-list, diff, add, commit, push) to find changed *.es.md files and push translated siblings back, but the python:3.12-slim image the translate step runs in doesn't include the git binary, so the pipeline failed immediately with: FileNotFoundError: [Errno 2] No such file or directory: 'git' Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NizVpJ2dwzCbjCrTLCjeHn --- .woodpecker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 7a7dcc7..b4d8104 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -26,6 +26,9 @@ steps: GITEA_PUSH_TOKEN: from_secret: gitea_push_token commands: + # translate.py shells out to git (rev-list/diff/add/commit/push) — the + # slim image doesn't ship it. + - apt-get update -qq && apt-get install -qq -y --no-install-recommends git - pip install --quiet requests pyyaml - python scripts/translate.py From 986c066ae33f66bc454264c894d905c45473d7c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 16:34:25 +0000 Subject: [PATCH 4/4] Replace DeepL with a self-hosted translation engine The DeepL free tier is metered (it failed in production with HTTP 456 Quota exceeded) and would require every deployment of this platform to carry its own API account. Translation now runs on M2M100 418M (MIT) via CTranslate2, shipped inside the pipeline image: no key, no quota, and no content or visitor data leaving the server. Also restores the multi-source behaviour of the original WordPress plugin, which the Python port had narrowed to Spanish-only. Any of the three site languages can now be the authored original. Because any language can be a source, loop prevention is no longer structural and is now explicit: generated siblings carry `translated_from` and are never treated as sources, and the bot's own [skip-translate] commits are skipped outright (that marker was already being written but never read). Markup protection moves in-process now that DeepL's tag_handling=html is gone. Code blocks and raw HTML pass through untouched; link targets, inline code and protected community terms are masked with placeholders that are verified to survive the round trip, failing the pipeline rather than shipping corrupted text. Two fixes along the way: - Generated siblings no longer inherit the source's `slug`. They did, which meant the first retranslation of a WordPress-migrated post moved /de// onto /de// and destroyed the inbound link preservation wp-to-hugo.py exists for. - `manual_translation` now works from the CMS. Decap only ever exposed it on the source while the script read it on the target, so the toggle did nothing. It now means "hands off" on both sides. wp-to-hugo.py marks migrated Polylang siblings frozen, since those are human translations and regenerating them would replace them with weaker machine output. Adds --backfill for sources missing siblings, which also fixes the existing 404s on /de/page/acerca/ and /pt-br/page/contacto/. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NizVpJ2dwzCbjCrTLCjeHn --- .woodpecker.yml | 21 +- README.md | 84 ++++-- docker/translate/Dockerfile | 42 +++ docs/server-setup.md | 40 ++- scripts/translate.py | 313 +++++++++++---------- scripts/translation/__init__.py | 1 + scripts/translation/ctranslate_provider.py | 58 ++++ scripts/translation/markdown.py | 146 ++++++++++ scripts/translation/provider.py | 26 ++ scripts/wp-to-hugo.py | 23 +- 10 files changed, 555 insertions(+), 199 deletions(-) create mode 100644 docker/translate/Dockerfile create mode 100644 scripts/translation/__init__.py create mode 100644 scripts/translation/ctranslate_provider.py create mode 100644 scripts/translation/markdown.py create mode 100644 scripts/translation/provider.py diff --git a/.woodpecker.yml b/.woodpecker.yml index b4d8104..889037c 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,12 +1,16 @@ # Woodpecker CI: translate → build → deploy # # Triggered by the Gitea push webhook. The translate step replaces the old -# synchronous WordPress save_post hook: publishing in Decap is one git -# commit, translation happens here, asynchronously, and every DeepL output -# lands as a reviewable bot commit. +# synchronous WordPress save_post hook: publishing in Decap is one git commit, +# translation happens here, asynchronously, and every generated sibling lands +# as a reviewable bot commit. +# +# Translation is self-hosted — the model ships inside vienalatina/translate, +# so there is no API key and no third-party request. Build that image on the +# server before the first run: +# docker build -t vienalatina/translate:1 docker/translate # # Secrets to configure in Woodpecker (repo settings → secrets): -# deepl_api_key — DeepL free-tier key (500k chars/mo) # gitea_push_token — Gitea token for the translations bot user # (translations@vienalatina.com) with repo write access # @@ -19,17 +23,12 @@ when: steps: translate: - image: python:3.12-slim + image: vienalatina/translate:1 + pull: false # built locally on the server, never fetched from a registry environment: - DEEPL_API_KEY: - from_secret: deepl_api_key GITEA_PUSH_TOKEN: from_secret: gitea_push_token commands: - # translate.py shells out to git (rev-list/diff/add/commit/push) — the - # slim image doesn't ship it. - - apt-get update -qq && apt-get install -qq -y --no-install-recommends git - - pip install --quiet requests pyyaml - python scripts/translate.py build: diff --git a/README.md b/README.md index 915e8b4..cbedf25 100644 --- a/README.md +++ b/README.md @@ -5,30 +5,39 @@ Vienna — Hugo + Decap CMS + Gitea + Woodpecker CI on a single Hetzner CX22 (Nuremberg, DE). Replaces the previous WordPress + Polylang + synchronous DeepL stack. +Translation is self-hosted too: M2M100 418M (MIT) runs on CPU via CTranslate2 +inside the pipeline image. No API key, no quota, and no visitor or content data +leaving the server. + ``` Pablo ──► Decap CMS (/admin) ──commit──► Gitea ──webhook──► Woodpecker CI │ - translate (DeepL, async) ─┤ - build (hugo) ─┤ - deploy (rsync) ─┘ + translate (M2M100, async) ──┤ + build (hugo) ─┤ + deploy (rsync) ─┘ ▼ Caddy 2 serves /var/www/vienalatina.com ``` ## Languages -Spanish is the authoring language; German (`/de/`) and Brazilian Portuguese -(`/pt-br/`) siblings are generated by CI as reviewable git commits. Siblings -pair by filename basename: +Spanish, German (`/de/`) and Brazilian Portuguese (`/pt-br/`). **Any of the +three can be the authored original**; the other two are generated by CI as +reviewable git commits. Siblings pair by filename basename: ``` -content/post/mi-articulo.es.md ← authored in Decap +content/post/mi-articulo.es.md ← authored (no `translated_from`) content/post/mi-articulo.de.md ← written by scripts/translate.py content/post/mi-articulo.pt-br.md ← written by scripts/translate.py ``` -Set `manual_translation: true` in a sibling's frontmatter to freeze it — -CI will never overwrite it again. +A generated file carries `translated_from`, which is what stops CI translating +its own output back into a loop. Machine output is never treated as a source. + +`manual_translation: true` means "hands off", on both sides: + +- on an **authored original** — don't generate siblings for this post at all +- on a **generated sibling** — never overwrite it again ### Frontmatter contract @@ -37,23 +46,59 @@ CI will never overwrite it again. title: "Mi artículo" date: 2026-07-31 lang: es -manual_translation: false # set true on *.de.md/*.pt-br.md to freeze -categories: [Gastronomía] +manual_translation: false +categories: [Gastronomía] # taxonomy terms stay Spanish in every language --- ``` +Generated siblings additionally carry `translated_from: es`. + ## Translation flow -1. Publish `*.es.md` in Decap → one git commit, no waiting on DeepL. +1. Publish in Decap → one git commit. 2. Gitea webhook fires Woodpecker. -3. `scripts/translate.py` diffs the push, translates changed `*.es.md` via - DeepL (`tag_handling=html`, protected community terms), writes the - siblings, and pushes them back as a bot commit - (`translations@vienalatina.com`). +3. `scripts/translate.py` diffs the push, finds changed authored files in any + language, generates the missing siblings, and pushes them back as a bot + commit (`translations@vienalatina.com`) marked `[skip-translate]`. 4. `hugo --minify` builds, `rsync --delete` deploys, Caddy serves. -Any DeepL error fails the pipeline visibly (red X, one-click retry) — -no half-translated sets ever ship. +Markup never reaches the model: code blocks and raw HTML pass through +untouched, and link targets, inline code and protected community terms +(`Grätzl`, `Naschmarkt`, …) are masked and verified to survive the round trip. +A mask that doesn't come back fails the pipeline rather than shipping corrupted +text — no half-translated sets ever ship. + +Backfill anything missing siblings (after the WP migration, or for pages that +predate the pipeline): + +```sh +python scripts/translate.py --backfill +``` + +## Translation engine + +Built once on the server, and again only when changing models: + +```sh +docker build -t vienalatina/translate:1 docker/translate +``` + +The image bakes in a pre-converted CTranslate2 build of M2M100 418M plus its +tokenizer, so a publish makes no network calls. Tunable via `MT_MODEL_DIR`, +`MT_TOKENIZER`, `MT_COMPUTE_TYPE` and `MT_THREADS`. + +Swapping engines means writing one class against +`scripts/translation/provider.py` — nothing in the pipeline changes. Two +upgrades worth knowing about: + +- **M2M100 1.2B** — same MIT licence and same code path, materially better + output, but ~2–2.5GB peak RAM. Needs an 8GB box (Hetzner CX32), not the CX22. +- **opus-mt-tc-big** — better still for these specific language pairs and the + only permissive option that handles Brazilian Portuguese distinctly (`>>pob<<`), + at the cost of CC-BY-4.0 attribution and one model per directed pair. + +Do **not** build on NLLB-200 (CC-BY-NC, non-commercial) or LibreTranslate +(AGPL-3.0) if this stack is ever to be sold or offered as a service. ## Server setup @@ -70,9 +115,10 @@ hugo server # http://localhost:1313 | Secret | Value | |---|---| -| `deepl_api_key` | DeepL free-tier key (500k chars/mo) | | `gitea_push_token` | Gitea token for the translations bot user, repo write access | +Translation needs no secret — the model is local. + The repo must be marked **trusted** in Woodpecker so the deploy step can mount `/var/www/vienalatina.com`. diff --git a/docker/translate/Dockerfile b/docker/translate/Dockerfile new file mode 100644 index 0000000..43ef7ee --- /dev/null +++ b/docker/translate/Dockerfile @@ -0,0 +1,42 @@ +# Pipeline image for the translate step. +# +# Model and tokenizer are baked in, so a publish makes zero network calls and +# needs no API key. torch is deliberately absent: it is only required to +# *convert* a model, and the default PyPI wheel drags in ~2.5GB of CUDA +# libraries this CPU-only box will never use. +# +# Build on the server (once, and again only when changing models): +# docker build -t vienalatina/translate:1 docker/translate +# +# The default model is a pre-converted CTranslate2 build, which avoids running +# ct2-transformers-converter on a 4GB box — it gets OOM-killed there. + +FROM python:3.12-slim + +ARG MT_MODEL_REPO=michaelfeil/ct2fast-m2m100_418M +ARG MT_TOKENIZER_REPO=facebook/m2m100_418M + +# translate.py shells out to git to diff the push and commit the siblings back. +RUN apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir \ + ctranslate2 \ + transformers \ + sentencepiece \ + sentencex \ + pyyaml \ + huggingface_hub + +RUN python -c "from huggingface_hub import snapshot_download; \ +snapshot_download('${MT_MODEL_REPO}', local_dir='/opt/mt/model')" \ + && python -c "import transformers; \ +transformers.AutoTokenizer.from_pretrained('${MT_TOKENIZER_REPO}') \ + .save_pretrained('/opt/mt/tokenizer')" + +# The published artifact is float16; CTranslate2 quantises to int8 on load. +ENV MT_MODEL_DIR=/opt/mt/model \ + MT_TOKENIZER=/opt/mt/tokenizer \ + MT_COMPUTE_TYPE=int8 \ + MT_THREADS=2 diff --git a/docs/server-setup.md b/docs/server-setup.md index 361e76e..e47358a 100644 --- a/docs/server-setup.md +++ b/docs/server-setup.md @@ -196,15 +196,23 @@ In Woodpecker (**https://ci.vienalatina.com**): (this auto-creates the push webhook in Gitea). 2. Repo → Settings → *Project settings* → check **Trusted** (needed so the deploy step may mount `/var/www/vienalatina.com`). -3. Repo → Settings → *Secrets* → add: - - `deepl_api_key` — your DeepL key (the same one from the WP plugin - settings page). - - `gitea_push_token` — the bot token from step 6.3. +3. Repo → Settings → *Secrets* → add `gitea_push_token`, the bot token from + step 6.3. That is the only secret — translation runs locally and needs no key. -The push in the step above has already triggered a first pipeline — it likely -ran before the secrets existed, so open it and press the retry button. All -three steps (translate → build → deploy) should go green, and -`/var/www/vienalatina.com/` on the server now contains the built site: +Build the translation image before the first run (~5 minutes; it downloads +about 1GB of model): + +```sh +cd ~/vienalatina +docker build -t vienalatina/translate:1 docker/translate +``` + +The push in the step above has already triggered a first pipeline — it ran +before the image and secret existed, so **push a new commit rather than using +Restart**. Restart replays the old commit, and a restart's empty diff range +makes the translate step find nothing to do. All three steps (translate → +build → deploy) should go green, and `/var/www/vienalatina.com/` on the server +now contains the built site: ```sh ls /var/www/vienalatina.com # index.html, de/, pt-br/, robots.txt, llms.txt … @@ -247,8 +255,14 @@ Saturday complete. 🎉 ## Cutover day (Sunday evening) 1. Run the content migration and push (see README, "One-shot content - migration"), spot-check the built site by IP or with - `curl -H "Host: vienalatina.com" http://127.0.0.1/...` on the server. + migration"). Migrated Polylang siblings arrive frozen + (`manual_translation: true`) because they are *human* translations — the + machine engine must never overwrite them. Then + `python scripts/translate.py --backfill` fills in any set that WordPress + had no translation for. Spot-check the built site with + `grep` on `/var/www/vienalatina.com/index.html`; the + `curl -H "Host: vienalatina.com" http://127.0.0.1/` trick only works after + step 3, since Caddy has no matching site block until then. 2. At the registrar: lower the `vienalatina.com` A record TTL to 300, wait for the old TTL to expire, then change the A record to `` (and `www` too, as CNAME to `vienalatina.com` or A to the same IP). @@ -256,8 +270,10 @@ Saturday complete. 🎉 `/etc/caddy/Caddyfile`, then `sudo systemctl reload caddy`. Caddy fetches the certificate as soon as DNS resolves to this server. 4. Verify: the checklist in the migration plan (hreflang tags, robots.txt, - llms.txt, Lighthouse, red-pipeline DeepL failure test, - `manual_translation: true` freeze test). + llms.txt, Lighthouse, `manual_translation: true` freeze test, and the + loop-prevention test — after the bot pushes siblings, the pipeline it + triggers must report "nothing to translate" rather than translating the + siblings back). 5. Keep the WP host untouched for 30 days as fallback; watch Google Search Console and add Caddy 301s for any 404s it reports. diff --git a/scripts/translate.py b/scripts/translate.py index e2f771f..1b684b7 100644 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -1,65 +1,57 @@ #!/usr/bin/env python3 -"""Async DeepL translation step for Woodpecker CI. +"""Self-hosted translation step for Woodpecker CI. -Replaces the synchronous `save_post` hook from the WordPress plugin -(plataforma_deepl_translate in plugin/plataforma-social/plataforma-social.php). +Replaces the DeepL API call this script used to make, and restores the +multi-source behaviour of the original WordPress plugin: any of the three site +languages may be the authored original, and the other two are generated from +it. Translation runs on a model shipped inside the pipeline image, so there is +no API key, no quota and no third-party request. -For each *.es.md changed in the pushed commit range, calls DeepL with -tag_handling=html (markup survives translation) plus a glossary of -community-specific terms, and writes the *.de.md and *.pt-br.md siblings -next to the source. Sibling files carrying `manual_translation: true` in -their frontmatter are never overwritten. The siblings are committed back -to the same branch as a bot commit, then the pipeline builds and deploys. +Loop prevention, which was structural back when only Spanish could be a source: + * a generated sibling carries `translated_from`, and a file carrying it is + never itself treated as a source; + * the bot's own commits carry [skip-translate] and are skipped outright. -Failure behaviour: any DeepL error exits non-zero, the pipeline goes red, +`manual_translation: true` means "hands off", on both sides: + * on an authored source — do not generate siblings for this post at all; + * on a generated sibling — never overwrite it again. + +Failure behaviour: any translation error exits non-zero, the pipeline goes red, and nothing partial is committed — no half-translated sets. Environment: - DEEPL_API_KEY required (Woodpecker secret) - DEEPL_API_URL optional, defaults to the free-tier endpoint - CI_COMMIT_SHA / CI_PREV_COMMIT_SHA provided by Woodpecker + MT_MODEL_DIR / MT_TOKENIZER / MT_COMPUTE_TYPE / MT_THREADS see translation/ + CI_COMMIT_SHA / CI_PREV_COMMIT_SHA / CI_COMMIT_MESSAGE from Woodpecker + GITEA_PUSH_TOKEN bot push token -Dependencies: requests, pyyaml +Dependencies: ctranslate2, transformers, sentencepiece, sentencex, pyyaml """ from __future__ import annotations +import argparse import os import re import subprocess -import sys from pathlib import Path -import requests import yaml +from translation.ctranslate_provider import CTranslate2Provider +from translation.markdown import translate_markdown, translate_text +from translation.provider import SITE_LANGS + REPO_ROOT = Path(__file__).resolve().parent.parent -DEEPL_URL = os.environ.get("DEEPL_API_URL", "https://api-free.deepl.com/v2/translate") +CONTENT_DIR = REPO_ROOT / "content" -# Language slug -> DeepL target code (ported from the plugin's $lang_map). -TARGETS = { - "de": "DE", - "pt-br": "PT-BR", -} -SOURCE_SLUG = "es" -DEEPL_SOURCE = "ES" - -# Fixed community-specific terms DeepL must not "translate". -# Sent as ignored tags via tag_handling=html: each term is wrapped in -# … before the call and unwrapped after, which pins proper -# nouns without needing a server-side DeepL glossary resource. -PROTECTED_TERMS = [ - "Viena Latina", - "Grätzl", - "empanadas de viento", - "Naschmarkt", -] - -# Frontmatter keys whose string values get translated alongside the body. +# Frontmatter strings translated alongside the body. Note `categories` is +# deliberately absent: the taxonomy terms stay Spanish in every language, or +# Hugo would fork the taxonomy per language. TRANSLATED_KEYS = ("title", "description") BOT_NAME = "vienalatina-translations" BOT_EMAIL = "translations@vienalatina.com" +SKIP_MARKER = "[skip-translate]" def run(*args: str, check: bool = True) -> str: @@ -67,33 +59,15 @@ def run(*args: str, check: bool = True) -> str: return result.stdout.strip() -def changed_source_files() -> list[Path]: - """Spanish sources touched by the pushed commits. - - Filtering to *.es.md is what makes bot commits (which only add .de.md / - .pt-br.md) a no-op round — the re-fire loop the WP hook had to guard - against with meta flags cannot happen here. - """ - head = os.environ.get("CI_COMMIT_SHA", "HEAD") - prev = os.environ.get("CI_PREV_COMMIT_SHA", "") - if prev and not set(prev) <= {"0"}: - diff_range = [prev, head] - else: - diff_range = ["HEAD~1", "HEAD"] if run("git", "rev-list", "--count", "HEAD") != "1" else None - - if diff_range: - out = run("git", "diff", "--name-only", "--diff-filter=AM", *diff_range) - else: # very first commit in the repo: translate everything - out = run("git", "ls-files") - - files = [] - for line in out.splitlines(): - p = Path(line.strip()) - if p.suffix == ".md" and p.name.endswith(f".{SOURCE_SLUG}.md") and p.parts[:1] == ("content",): - full = REPO_ROOT / p - if full.exists(): - files.append(full) - return files +def split_lang(path: Path) -> tuple[str, str] | None: + """('mi-articulo', 'es') for mi-articulo.es.md, else None.""" + if path.suffix != ".md": + return None + stem = path.name[: -len(".md")] + for lang in SITE_LANGS: + if stem.endswith(f".{lang}"): + return stem[: -(len(lang) + 1)], lang + return None def split_frontmatter(text: str) -> tuple[dict, str]: @@ -108,93 +82,109 @@ def join_frontmatter(fm: dict, body: str) -> str: return f"---\n{front}---\n\n{body.lstrip()}" -def protect(text: str) -> str: - for term in PROTECTED_TERMS: - text = re.sub(re.escape(term), lambda m: f"{m.group(0)}", text, flags=re.IGNORECASE) - return text - - -def unprotect(text: str) -> str: - return re.sub(r"", "", text) - - -def deepl_translate(texts: list[str], target: str, html: bool) -> list[str]: - """Direct port of plataforma_deepl_translate(): same endpoint, same - tag_handling=html behaviour, but errors abort the pipeline instead of - silently shipping a half-translated post.""" - texts = [t for t in texts if isinstance(t, str) and t.strip()] - if not texts: - return [] - - key = os.environ.get("DEEPL_API_KEY", "") - if not key: - sys.exit("DEEPL_API_KEY is not set — configure the Woodpecker secret.") - - data: list[tuple[str, str]] = [ - ("target_lang", target), - ("source_lang", DEEPL_SOURCE), - ("tag_handling", "html"), - ("ignore_tags", "keep"), - ] - if not html: - # Titles/descriptions are plain strings; still use tag handling so - # protection works, DeepL just has no other tags to preserve. - pass - for t in texts: - data.append(("text", protect(t))) - - resp = requests.post( - DEEPL_URL, - headers={"Authorization": f"DeepL-Auth-Key {key}"}, - data=data, - timeout=60, - ) - if resp.status_code != 200: - sys.exit(f"DeepL HTTP {resp.status_code}: {resp.text[:300]}") - - return [unprotect(item["text"]) for item in resp.json().get("translations", [])] - - -def sibling_path(source: Path, slug: str) -> Path: - return source.with_name(source.name.replace(f".{SOURCE_SLUG}.md", f".{slug}.md")) - - -def is_frozen(path: Path) -> bool: +def read_frontmatter(path: Path) -> dict: if not path.exists(): - return False + return {} fm, _ = split_frontmatter(path.read_text(encoding="utf-8")) + return fm + + +def is_generated(fm: dict) -> bool: + return bool(fm.get("translated_from")) + + +def is_frozen(fm: dict) -> bool: return bool(fm.get("manual_translation")) -def translate_file(source: Path) -> list[Path]: - raw = source.read_text(encoding="utf-8") - fm, body = split_frontmatter(raw) - written = [] +def changed_markdown() -> list[Path]: + """Content files touched by the pushed commits.""" + head = os.environ.get("CI_COMMIT_SHA", "HEAD") + prev = os.environ.get("CI_PREV_COMMIT_SHA", "") + if prev and not set(prev) <= {"0"}: + out = run("git", "diff", "--name-only", "--diff-filter=AM", prev, head) + elif run("git", "rev-list", "--count", "HEAD") != "1": + out = run("git", "diff", "--name-only", "--diff-filter=AM", "HEAD~1", "HEAD") + else: # first commit in the repo + out = run("git", "ls-files") - for slug, deepl_target in TARGETS.items(): - target_file = sibling_path(source, slug) - if is_frozen(target_file): - print(f" {target_file.relative_to(REPO_ROOT)}: manual_translation=true — skipped") + paths = [] + for line in out.splitlines(): + rel = Path(line.strip()) + if rel.parts[:1] == ("content",) and (REPO_ROOT / rel).exists(): + paths.append(REPO_ROOT / rel) + return paths + + +def authored_sources(paths: list[Path]) -> list[tuple[Path, str, str]]: + """(path, basename, lang) for files that may act as a translation source.""" + sources = [] + for path in paths: + parsed = split_lang(path) + if not parsed: + continue + basename, lang = parsed + fm = read_frontmatter(path) + if is_generated(fm): + continue # machine output is never a source + if is_frozen(fm): + print(f"{path.relative_to(REPO_ROOT)}: manual_translation=true — not translated") + continue + sources.append((path, basename, lang)) + return sources + + +def missing_siblings() -> list[Path]: + """Every authored source missing at least one sibling.""" + incomplete = [] + for path in sorted(CONTENT_DIR.rglob("*.md")): + parsed = split_lang(path) + if not parsed: + continue + basename, lang = parsed + if is_generated(read_frontmatter(path)): + continue + for target in SITE_LANGS: + if target != lang and not path.with_name(f"{basename}.{target}.md").exists(): + incomplete.append(path) + break + return incomplete + + +def translate_file(source: Path, basename: str, src_lang: str, provider) -> list[Path]: + fm, body = split_frontmatter(source.read_text(encoding="utf-8")) + written: list[Path] = [] + + for tgt in SITE_LANGS: + if tgt == src_lang: + continue + target = source.with_name(f"{basename}.{tgt}.md") + target_fm = read_frontmatter(target) + if is_frozen(target_fm): + print(f" {target.relative_to(REPO_ROOT)}: manual_translation=true — skipped") continue - strings = [str(fm[k]) for k in TRANSLATED_KEYS if fm.get(k)] - translated_strings = deepl_translate(strings, deepl_target, html=False) - translated_body = deepl_translate([body], deepl_target, html=True) if body.strip() else [""] - new_fm = dict(fm) - it = iter(translated_strings) - for k in TRANSLATED_KEYS: - if fm.get(k): - new_fm[k] = next(it) - new_fm["lang"] = slug + for key in TRANSLATED_KEYS: + if fm.get(key): + new_fm[key] = translate_text(str(fm[key]), src_lang, tgt, provider) + new_fm["lang"] = tgt + new_fm["translated_from"] = src_lang new_fm["manual_translation"] = False - # Pin the URL to the shared basename so it never drifts when a title - # is retranslated (plan: /de/mi-articulo/ pairs with /mi-articulo/). - new_fm.setdefault("slug", source.name.removesuffix(f".{SOURCE_SLUG}.md")) - target_file.write_text(join_frontmatter(new_fm, translated_body[0] if translated_body else ""), encoding="utf-8") - written.append(target_file) - print(f" {target_file.relative_to(REPO_ROOT)}: written") + # Never inherit the source's slug: that would move a migrated sibling's + # URL onto the source's and break inbound links. Keep the slug the + # target already had; otherwise let Hugo fall back to the filename. + new_fm.pop("slug", None) + if target_fm.get("slug"): + new_fm["slug"] = target_fm["slug"] + + target.write_text( + join_frontmatter(new_fm, translate_markdown(body, src_lang, tgt, provider)), + encoding="utf-8", + ) + written.append(target) + print(f" {target.relative_to(REPO_ROOT)}: written") return written @@ -205,30 +195,49 @@ def commit_and_push(files: list[Path]) -> None: run("git", "config", "user.email", BOT_EMAIL) token = os.environ.get("GITEA_PUSH_TOKEN", "") repo = os.environ.get("CI_REPO", "pablo/vienalatina") - if token: # Woodpecker's clone credentials are read-only; push needs its own token - run("git", "remote", "set-url", "origin", f"https://{BOT_NAME}:{token}@git.vienalatina.com/{repo}.git") + if token: # clone credentials are read-only; pushing needs the bot token + run("git", "remote", "set-url", "origin", + f"https://{BOT_NAME}:{token}@git.vienalatina.com/{repo}.git") run("git", "add", *[str(f) for f in files]) if not run("git", "status", "--porcelain"): print("Translations identical to committed siblings — nothing to push.") return - run("git", "commit", "-m", "translate: update DE and PT-BR siblings [skip-translate]") + run("git", "commit", "-m", f"translate: update generated siblings {SKIP_MARKER}") run("git", "push", "origin", f"HEAD:{branch}") print(f"Pushed sibling commit to {branch}.") def main() -> None: - sources = changed_source_files() - if not sources: - print("No changed *.es.md files — nothing to translate.") + parser = argparse.ArgumentParser(description="Generate translated content siblings.") + parser.add_argument("--backfill", action="store_true", + help="translate every source missing siblings, not just changed files") + parser.add_argument("--no-push", action="store_true", + help="write siblings but do not commit or push (local testing)") + args = parser.parse_args() + + if SKIP_MARKER in os.environ.get("CI_COMMIT_MESSAGE", ""): + print(f"{SKIP_MARKER} commit — nothing to translate.") return - all_written: list[Path] = [] - for source in sources: - print(f"Translating {source.relative_to(REPO_ROOT)}:") - all_written.extend(translate_file(source)) + paths = missing_siblings() if args.backfill else changed_markdown() + sources = authored_sources(paths) + if not sources: + print("No authored content changed — nothing to translate.") + return - if all_written: - commit_and_push(all_written) + provider = CTranslate2Provider() + written: list[Path] = [] + for source, basename, lang in sources: + print(f"Translating {source.relative_to(REPO_ROOT)} (from {lang}):") + written.extend(translate_file(source, basename, lang, provider)) + + if not written: + print("Every sibling is frozen — nothing written.") + return + if args.no_push: + print(f"--no-push: wrote {len(written)} files, leaving them uncommitted.") + return + commit_and_push(written) if __name__ == "__main__": diff --git a/scripts/translation/__init__.py b/scripts/translation/__init__.py new file mode 100644 index 0000000..cc4ea7b --- /dev/null +++ b/scripts/translation/__init__.py @@ -0,0 +1 @@ +"""Self-hosted translation for the Viena Latina pipeline.""" diff --git a/scripts/translation/ctranslate_provider.py b/scripts/translation/ctranslate_provider.py new file mode 100644 index 0000000..6960ed6 --- /dev/null +++ b/scripts/translation/ctranslate_provider.py @@ -0,0 +1,58 @@ +"""M2M100 via CTranslate2, on CPU. + +Replaces the DeepL HTTP call. No API key, no quota, no third-party request — +the model ships inside the pipeline image. +""" + +from __future__ import annotations + +import os + +from .provider import Provider, SITE_TO_MODEL + + +class CTranslate2Provider(Provider): + def __init__( + self, + model_dir: str | None = None, + tokenizer: str | None = None, + compute_type: str | None = None, + threads: int | None = None, + ): + self.model_dir = model_dir or os.environ.get("MT_MODEL_DIR", "/opt/mt/model") + self.tokenizer = tokenizer or os.environ.get("MT_TOKENIZER", "facebook/m2m100_418M") + self.compute_type = compute_type or os.environ.get("MT_COMPUTE_TYPE", "int8") + self.threads = threads or int(os.environ.get("MT_THREADS", "2")) + self._translator = None + self._tokenizer = None + + def _load(self) -> None: + if self._translator is not None: + return + import ctranslate2 + import transformers + + self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.tokenizer) + self._translator = ctranslate2.Translator( + self.model_dir, + device="cpu", + compute_type=self.compute_type, + intra_threads=self.threads, + ) + + def translate(self, texts: list[str], src: str, tgt: str) -> list[str]: + if not texts: + return [] + self._load() + tok = self._tokenizer + tok.src_lang = SITE_TO_MODEL[src] + target_token = tok.lang_code_to_token[SITE_TO_MODEL[tgt]] + + batch = [tok.convert_ids_to_tokens(tok.encode(t)) for t in texts] + results = self._translator.translate_batch( + batch, target_prefix=[[target_token]] * len(batch) + ) + # hypotheses[0][0] is the target-language token we forced; drop it. + return [ + tok.decode(tok.convert_tokens_to_ids(r.hypotheses[0][1:])) for r in results + ] diff --git a/scripts/translation/markdown.py b/scripts/translation/markdown.py new file mode 100644 index 0000000..ddda525 --- /dev/null +++ b/scripts/translation/markdown.py @@ -0,0 +1,146 @@ +"""Markdown-safe translation. + +DeepL preserved markup server-side with tag_handling=html. A self-hosted NMT +model has no equivalent, so structure is protected here instead: non-prose +blocks pass through untouched, and inline constructs are masked with opaque +placeholders whose survival is verified after the round trip. + +Placeholders use OpenNMT's protected-sequence convention (U+FF5F/U+FF60). +SentencePiece keeps these atomic; ``{{x}}``, ```` and ``%s`` get fragmented +by BPE and dropped by the model. +""" + +from __future__ import annotations + +import re + +from .provider import SITE_TO_MODEL, Provider + +OPEN, CLOSE = "｟", "｠" + +# Community vocabulary that must reach readers unchanged. Not inherited from +# the WordPress plugin, which had no glossary at all — edit freely. +PROTECTED_TERMS = [ + "Viena Latina", + "Grätzl", + "empanadas de viento", + "Naschmarkt", +] + +_FENCE = re.compile(r"^\s*(?:```|~~~)") +_HEADING = re.compile(r"^(#{1,6}\s+)(.*)$") +_LIST = re.compile(r"^(\s*(?:[-*+]|\d+[.)])\s+)(.*)$") +_QUOTE = re.compile(r"^(\s*>\s?)(.*)$") +_HTML_BLOCK = re.compile(r"^\s*<") +_PREFIXED = (_HEADING, _LIST, _QUOTE) + +# Inline spans that must never reach the model. Order matters: inline code is +# taken first so a URL inside backticks is masked once, not twice. +_INLINE = ( + re.compile(r"`[^`]*`"), # inline code + re.compile(r"\]\([^)]*\)"), # link/image target — the label stays translatable + re.compile(r"<[^>\s][^>]*>"), # raw HTML tags, autolinks + re.compile(r"https?://\S+"), # bare URLs +) + +_PLACEHOLDER = re.compile(re.escape(OPEN) + r"\s*(\d+)\s*" + re.escape(CLOSE)) + + +class PlaceholderError(RuntimeError): + """A masked span did not survive translation intact.""" + + +class _Masker: + def __init__(self) -> None: + self.spans: list[str] = [] + + def _take(self, match: re.Match) -> str: + self.spans.append(match.group(0)) + return f"{OPEN}{len(self.spans) - 1}{CLOSE}" + + def mask(self, text: str) -> str: + for pattern in _INLINE: + text = pattern.sub(self._take, text) + for term in PROTECTED_TERMS: + text = re.sub(re.escape(term), self._take, text, flags=re.IGNORECASE) + return text + + def restore(self, text: str) -> str: + # Models pad and reorder placeholders; normalise spacing before matching. + text = _PLACEHOLDER.sub(lambda m: f"{OPEN}{m.group(1)}{CLOSE}", text) + for index, span in enumerate(self.spans): + token = f"{OPEN}{index}{CLOSE}" + seen = text.count(token) + if seen != 1: + raise PlaceholderError( + f"masked span {span!r} came back {seen} times, expected once" + ) + text = text.replace(token, span) + return text + + +def _sentences(text: str, lang: str) -> list[str]: + from sentencex import segment + + return [s.strip() for s in segment(SITE_TO_MODEL[lang], text) if s.strip()] + + +def translate_text(text: str, src: str, tgt: str, provider: Provider) -> str: + """Translate one prose string, protecting inline markup and fixed terms.""" + if not text.strip(): + return text + masker = _Masker() + pieces = _sentences(masker.mask(text), src) + if not pieces: + return text + return masker.restore(" ".join(provider.translate(pieces, src, tgt))) + + +def _is_prose(line: str) -> bool: + return bool( + line.strip() + and not _FENCE.match(line) + and not _HTML_BLOCK.match(line) + and not any(p.match(line) for p in _PREFIXED) + ) + + +def translate_markdown(body: str, src: str, tgt: str, provider: Provider) -> str: + """Translate a markdown body, leaving every non-prose construct intact.""" + lines = body.split("\n") + out: list[str] = [] + i = 0 + + while i < len(lines): + line = lines[i] + + if _FENCE.match(line): + out.append(line) + i += 1 + while i < len(lines) and not _FENCE.match(lines[i]): + out.append(lines[i]) + i += 1 + if i < len(lines): + out.append(lines[i]) + i += 1 + continue + + if not line.strip() or _HTML_BLOCK.match(line): + out.append(line) + i += 1 + continue + + prefixed = next((m for m in (p.match(line) for p in _PREFIXED) if m), None) + if prefixed: + out.append(prefixed.group(1) + translate_text(prefixed.group(2), src, tgt, provider)) + i += 1 + continue + + # A soft-wrapped paragraph: rejoin it so sentences are translated whole. + para: list[str] = [] + while i < len(lines) and _is_prose(lines[i]): + para.append(lines[i].strip()) + i += 1 + out.append(translate_text(" ".join(para), src, tgt, provider)) + + return "\n".join(out) diff --git a/scripts/translation/provider.py b/scripts/translation/provider.py new file mode 100644 index 0000000..d5d3966 --- /dev/null +++ b/scripts/translation/provider.py @@ -0,0 +1,26 @@ +"""Translation provider interface. + +The pipeline talks to a provider, never to a model directly, so the engine can +be swapped — a larger M2M100, OPUS-MT, or a hosted API — without touching +translate.py. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +# Site language codes (Hugo, and the ..md filename suffix) -> model codes. +# M2M100 has no Brazilian variant, so pt-br translates as generic Portuguese. +SITE_TO_MODEL = {"es": "es", "de": "de", "pt-br": "pt"} + +SITE_LANGS = tuple(SITE_TO_MODEL) + + +class Provider(ABC): + @abstractmethod + def translate(self, texts: list[str], src: str, tgt: str) -> list[str]: + """Translate plain-text strings between two site language codes. + + Input must carry no markup: callers mask it first (see markdown.py). + Returns one string per input, in order. + """ diff --git a/scripts/wp-to-hugo.py b/scripts/wp-to-hugo.py index d56a57f..ce6bc33 100644 --- a/scripts/wp-to-hugo.py +++ b/scripts/wp-to-hugo.py @@ -81,7 +81,8 @@ def to_markdown(html: str) -> str: return conv.handle(html).strip() -def frontmatter(post: dict, lang: str, categories: dict[int, str]) -> str: +def frontmatter(post: dict, lang: str, categories: dict[int, str], + derived_from: str | None = None) -> str: title = to_markdown(post["title"]["rendered"]).replace('"', '\\"') cats = [categories[c] for c in post.get("categories", []) if c in categories] lines = [ @@ -90,9 +91,15 @@ def frontmatter(post: dict, lang: str, categories: dict[int, str]) -> str: f"date: {post['date']}", f"slug: {post['slug']}", # preserve the exact WP slug per language f"lang: {lang}", - "manual_translation: false", - f"categories: [{', '.join(cats)}]", ] + if derived_from: + # A Polylang sibling: derived, so translate.py never treats it as a + # source — and frozen, because these are *human* WordPress translations + # and regenerating them would replace them with weaker machine output. + lines.extend([f"translated_from: {derived_from}", "manual_translation: true"]) + else: + lines.append("manual_translation: false") + lines.append(f"categories: [{', '.join(cats)}]") excerpt = to_markdown(post.get("excerpt", {}).get("rendered", "")) if excerpt: lines.append(f'description: "{excerpt[:300].replace(chr(34), chr(39))}"') @@ -125,10 +132,16 @@ def main() -> None: # sibling's slug as the shared basename so Hugo pairs the set. translations = post.get("translations", {}) es_id = translations.get("es", post["id"]) - basename = by_id.get(es_id, post)["slug"] + origin = by_id.get(es_id, post) + basename = origin["slug"] + + # Anything that isn't the origin of its Polylang group is a translation. + origin_lang = LANG_MAP.get(origin.get("lang", "es")) + derived_from = origin_lang if origin_lang and origin_lang != lang else None body_html = rewrite_images(post["content"]["rendered"], base_url) - md = frontmatter(post, lang, categories) + "\n\n" + to_markdown(body_html) + "\n" + md = (frontmatter(post, lang, categories, derived_from) + + "\n\n" + to_markdown(body_html) + "\n") out = POSTS_DIR / f"{basename}.{lang}.md" out.write_text(md, encoding="utf-8")