341 lines
14 KiB
Python
341 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
translate_post.py — Traduce posts de feadulta (ES → EN/FR/IT/PT) con Gemma 4B local
|
|
y los enlaza como traducciones de Polylang, SIN servicios de pago.
|
|
|
|
Diseño (issue rafa/feadulta#75, fase 1):
|
|
- Gemma (LM Studio, http://172.19.128.1:1234/v1) traduce título + contenido HTML.
|
|
- Reglas estrictas: preserva HTML/shortcodes, NO traduce referencias bíblicas, respeta
|
|
nombres propios y un glosario fijo del proyecto. Traducción fiel (sin resumir).
|
|
- La lógica WordPress/Polylang vive en fea_translate_helper.php (corre dentro del
|
|
contenedor cargando wp-load.php; no necesita wp-cli ni proc_open).
|
|
- Idempotente y reanudable: si ya existe la traducción en ese idioma, se salta.
|
|
|
|
Uso:
|
|
python3 scripts/translate_post.py --post-id 45018 --langs en,fr,it,pt
|
|
python3 scripts/translate_post.py --carta 45018 # carta + sus _carta_id
|
|
python3 scripts/translate_post.py --post-id 45018 --langs en --status publish --force
|
|
|
|
Pensado para que Codex lo lance en lote sobre la cola priorizada (cartas/destacados).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
# ── Configuración ────────────────────────────────────────────────────────────
|
|
WP_CONTAINER = os.environ.get("FEA_WP_CONTAINER", "wordpress-web")
|
|
DB_CONTAINER = os.environ.get("FEA_DB_CONTAINER", "wordpress-mysql")
|
|
DB_NAME = os.environ.get("FEA_DB_NAME", "wordpress_db")
|
|
DB_USER = os.environ.get("FEA_DB_USER", "wordpress_user")
|
|
DB_PASS = os.environ.get("FEA_DB_PASS", "wordpress_pass")
|
|
|
|
LM_BASE_URL = os.environ.get("OPENAI_BASE_URL", "http://172.19.128.1:1234/v1")
|
|
MODEL = os.environ.get("LOCAL_MODEL", "google/gemma-4-e4b")
|
|
|
|
# Motor de traducción: "gemma" (local, por defecto) o "haiku" (Claude Haiku 4.5 vía API).
|
|
# Haiku da más calidad y no necesita trocear (contexto 200k). Reutiliza translate_haiku.py.
|
|
ENGINE = os.environ.get("FEA_ENGINE", "gemma").lower()
|
|
if ENGINE == "haiku":
|
|
MODEL = "claude-haiku-4-5"
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import translate_haiku # carga la API key de portfolio-tracker/.env
|
|
elif ENGINE == "minimax":
|
|
MODEL = os.environ.get("LOCAL_MODEL", "MiniMax-Text-01")
|
|
MINIMAX_URL = os.environ.get("MINIMAX_URL", "https://api.minimax.io/v1/text/chatcompletion_v2")
|
|
_kf = Path(os.environ.get("MINIMAX_KEY_FILE", "/home/rafa/Feadulta/minimax.txt"))
|
|
_keys = [l.strip() for l in _kf.read_text().splitlines() if l.strip().startswith("sk-")]
|
|
MINIMAX_KEY = _keys[-1] if _keys else ""
|
|
|
|
HELPER_SRC = Path(__file__).resolve().parent / "fea_translate_helper.php"
|
|
HELPER_DST = "/tmp/fea_translate_helper.php"
|
|
STATE_FILE = Path(os.environ.get("FEA_TR_STATE", "/tmp/feadulta-translate-state.json"))
|
|
LOG_FILE = Path(os.environ.get("FEA_TR_LOG", "/tmp/feadulta-translate.log"))
|
|
|
|
LANG_NAMES = {"en": "English", "fr": "French (français)", "it": "Italian (italiano)", "pt": "Portuguese (português)"}
|
|
|
|
# Glosario: términos que NO se traducen o se fijan.
|
|
GLOSSARY = {
|
|
"Fe Adulta": {"en": "Fe Adulta", "fr": "Fe Adulta", "it": "Fe Adulta", "pt": "Fe Adulta"},
|
|
"EFFA": {"en": "EFFA", "fr": "EFFA", "it": "EFFA", "pt": "EFFA"},
|
|
}
|
|
|
|
CHUNK_LIMIT = 5000 # caracteres por llamada a Gemma (parte por </p> si se supera)
|
|
|
|
|
|
# ── Utilidades de proceso ────────────────────────────────────────────────────
|
|
def log(msg: str) -> None:
|
|
line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}"
|
|
print(line, flush=True)
|
|
try:
|
|
LOG_FILE.open("a", encoding="utf-8").write(line + "\n")
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def sh(cmd: list[str], *, stdin: str | None = None, timeout: int = 120) -> str:
|
|
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, timeout=timeout)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f"cmd falló ({r.returncode}): {' '.join(cmd)}\n{r.stderr.strip()}")
|
|
return r.stdout
|
|
|
|
|
|
_helper_ready = False
|
|
|
|
|
|
def php_helper(subcmd: str, *args: str, stdin: str | None = None) -> str:
|
|
"""Copia el helper al contenedor (una vez) y lo ejecuta cargando wp-load.php."""
|
|
global _helper_ready
|
|
if not _helper_ready:
|
|
sh(["docker", "cp", str(HELPER_SRC), f"{WP_CONTAINER}:{HELPER_DST}"])
|
|
_helper_ready = True
|
|
cmd = ["docker", "exec", "-i", WP_CONTAINER, "php", HELPER_DST, subcmd, *args]
|
|
return sh(cmd, stdin=stdin, timeout=180)
|
|
|
|
|
|
# ── Gemma (LM Studio) ────────────────────────────────────────────────────────
|
|
def gemma(messages: list[dict], *, max_tokens: int) -> str:
|
|
import urllib.request
|
|
|
|
body = json.dumps({
|
|
"model": MODEL,
|
|
"messages": messages,
|
|
"temperature": 0.2,
|
|
"max_tokens": max_tokens,
|
|
"reasoning_effort": "none",
|
|
}).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
f"{LM_BASE_URL}/chat/completions", data=body,
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=300) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
return data["choices"][0]["message"]["content"]
|
|
|
|
|
|
def minimax(messages: list[dict], *, max_tokens: int) -> str:
|
|
import urllib.request
|
|
|
|
body = json.dumps({
|
|
"model": MODEL,
|
|
"messages": messages,
|
|
"temperature": 0.2,
|
|
"max_tokens": max_tokens,
|
|
}).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
MINIMAX_URL, data=body,
|
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {MINIMAX_KEY}"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=300) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
return data["choices"][0]["message"]["content"]
|
|
|
|
|
|
def _extract(text: str) -> str:
|
|
"""Extrae la traducción del ÚLTIMO bloque <<<INI>>>…<<<FIN>>>.
|
|
|
|
Gemma (modo reasoning) escribe un preámbulo que MENCIONA las propias marcas,
|
|
así que hay que quedarse con la última ocurrencia, no la primera.
|
|
"""
|
|
start_tok, end_tok = "<<<INI>>>", "<<<FIN>>>"
|
|
i = text.rfind(start_tok)
|
|
if i != -1:
|
|
rest = text[i + len(start_tok):]
|
|
j = rest.find(end_tok)
|
|
out = (rest[:j] if j != -1 else rest).strip()
|
|
else:
|
|
out = text.strip()
|
|
# Quita vallas de código markdown si Gemma las añade.
|
|
out = re.sub(r"^```[a-z]*\n?", "", out)
|
|
out = re.sub(r"\n?```$", "", out)
|
|
return out.strip()
|
|
|
|
|
|
def _system_prompt(lang: str) -> str:
|
|
target = LANG_NAMES[lang]
|
|
glos = "; ".join(f'"{k}" → "{v[lang]}"' for k, v in GLOSSARY.items())
|
|
return (
|
|
f"Eres un traductor profesional de textos religiosos cristianos (espiritualidad y "
|
|
f"teología católica). Traduce del español al {target}. REGLAS ESTRICTAS:\n"
|
|
f"1. Conserva EXACTAMENTE el marcado HTML (etiquetas y atributos) y los shortcodes "
|
|
f"entre [ ] y { '{' } { '}' }. No los traduzcas ni los reordenes.\n"
|
|
f"2. NO traduzcas las referencias bíblicas ni sus abreviaturas (p.ej. 'Jn 3, 16', "
|
|
f"'Isaías 5, 1-7', 'Mt 5'). Déjalas idénticas.\n"
|
|
f"3. Conserva los nombres propios de persona y lugar (salvo exónimos establecidos).\n"
|
|
f"4. Glosario fijo: {glos}.\n"
|
|
f"5. Traducción FIEL: no resumas, no añadas, no comentes.\n"
|
|
f"6. Devuelve SOLO la traducción entre las marcas <<<INI>>> y <<<FIN>>>, sin nada más."
|
|
)
|
|
|
|
|
|
def translate_text(text: str, lang: str, *, is_title: bool = False) -> str:
|
|
text = text.strip()
|
|
if not text:
|
|
return ""
|
|
if ENGINE == "haiku":
|
|
out, _usage = translate_haiku.translate(text, lang, is_title=is_title)
|
|
return out
|
|
user = f"<<<INI>>>{text}<<<FIN>>>"
|
|
if is_title:
|
|
kind = "el TÍTULO"
|
|
task = (
|
|
f"Traduce {kind} que va entre las marcas.\n"
|
|
f"Debe quedar en {LANG_NAMES[lang]} de forma natural. "
|
|
f"No lo dejes en inglés salvo que el original ya sea un nombre propio o una marca.\n"
|
|
f"Responde solo con el título traducido entre las marcas:\n{user}"
|
|
)
|
|
else:
|
|
kind = "el texto"
|
|
task = f"Traduce {kind} que va entre las marcas:\n{user}"
|
|
messages = [
|
|
{"role": "system", "content": _system_prompt(lang)},
|
|
{"role": "user", "content": task},
|
|
]
|
|
max_tokens = max(800, int(len(text) * 1.6))
|
|
engine_fn = minimax if ENGINE == "minimax" else gemma
|
|
raw = engine_fn(messages, max_tokens=max_tokens)
|
|
return _extract(raw)
|
|
|
|
|
|
def translate_html(html: str, lang: str) -> str:
|
|
"""Trocea por párrafos si el contenido es largo, para no saturar el contexto de Gemma."""
|
|
if ENGINE == "haiku":
|
|
# Haiku tiene 200k de contexto: el artículo entero de una vez (mejor coherencia).
|
|
return translate_text(html, lang)
|
|
if len(html) <= CHUNK_LIMIT:
|
|
return translate_text(html, lang)
|
|
parts = re.split(r"(?<=</p>)", html)
|
|
chunks, buf = [], ""
|
|
for p in parts:
|
|
if len(buf) + len(p) > CHUNK_LIMIT and buf:
|
|
chunks.append(buf)
|
|
buf = ""
|
|
buf += p
|
|
if buf:
|
|
chunks.append(buf)
|
|
log(f" contenido largo ({len(html)} car) → {len(chunks)} trozos")
|
|
return "".join(translate_text(c, lang) for c in chunks)
|
|
|
|
|
|
# ── Datos / estado ───────────────────────────────────────────────────────────
|
|
def read_post(post_id: int) -> dict:
|
|
return json.loads(php_helper("read", str(post_id)))
|
|
|
|
|
|
def translation_exists(es_id: int, lang: str) -> int:
|
|
return int(php_helper("exists", str(es_id), lang).strip() or "0")
|
|
|
|
|
|
WP_LOCK_FILE = Path(os.environ.get("FEA_TR_LOCK", "/tmp/feadulta-translate.lock"))
|
|
|
|
|
|
def create_translation(es_id: int, lang: str, title: str, content: str, status: str) -> int:
|
|
payload = json.dumps({"title": title, "content": content, "model": MODEL})
|
|
# Lock entre procesos: serializa SOLO la escritura/enlace Polylang (rápido), no la
|
|
# traducción LLM (lenta), para que 4 streams por idioma no pisen el grupo de traducciones.
|
|
with WP_LOCK_FILE.open("w") as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
try:
|
|
return int(php_helper("create", str(es_id), lang, status, stdin=payload).strip())
|
|
finally:
|
|
fcntl.flock(lk, fcntl.LOCK_UN)
|
|
|
|
|
|
def carta_article_ids(carta_id: int) -> list[int]:
|
|
q = (f"SELECT post_id FROM wp_postmeta WHERE meta_key='_carta_id' "
|
|
f"AND meta_value='{carta_id}' ORDER BY post_id;")
|
|
out = sh(["docker", "exec", DB_CONTAINER, "mysql", f"-u{DB_USER}", f"-p{DB_PASS}",
|
|
DB_NAME, "-N", "-e", q])
|
|
return [int(x) for x in out.split() if x.strip().isdigit()]
|
|
|
|
|
|
def load_state() -> dict:
|
|
if STATE_FILE.exists():
|
|
try:
|
|
return json.loads(STATE_FILE.read_text())
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return {"done": {}}
|
|
|
|
|
|
def save_state(state: dict) -> None:
|
|
STATE_FILE.write_text(json.dumps(state, ensure_ascii=False, indent=2))
|
|
|
|
|
|
# ── Orquestación ─────────────────────────────────────────────────────────────
|
|
def process_post(post_id: int, langs: list[str], status: str, force: bool, state: dict) -> None:
|
|
src = read_post(post_id)
|
|
if src.get("lang") and src["lang"] != "es":
|
|
log(f"#{post_id} no es ES (lang={src['lang']}) — saltado")
|
|
return
|
|
log(f"#{post_id} «{src['title'][:60]}»")
|
|
|
|
for lang in langs:
|
|
key = f"{post_id}:{lang}"
|
|
existing = translation_exists(post_id, lang)
|
|
if existing and not force:
|
|
log(f" {lang}: ya existe (#{existing}) — saltado")
|
|
state["done"][key] = existing
|
|
continue
|
|
if existing and force:
|
|
php_helper("unlink", str(post_id), lang)
|
|
log(f" {lang}: --force, eliminada traducción previa #{existing}")
|
|
try:
|
|
t0 = time.time()
|
|
title = translate_text(src["title"], lang, is_title=True)
|
|
content = translate_html(src["content"], lang)
|
|
new_id = create_translation(post_id, lang, title, content, status)
|
|
dt = time.time() - t0
|
|
log(f" {lang}: creado #{new_id} ({dt:.0f}s) → «{title[:50]}»")
|
|
state["done"][key] = new_id
|
|
save_state(state)
|
|
except Exception as exc: # noqa: BLE001
|
|
log(f" {lang}: ERROR {exc}")
|
|
state.setdefault("errors", {})[key] = str(exc)
|
|
save_state(state)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description="Traduce posts de feadulta con Gemma local + Polylang.")
|
|
g = ap.add_mutually_exclusive_group(required=True)
|
|
g.add_argument("--post-id", type=int, help="ID de un post ES a traducir.")
|
|
g.add_argument("--carta", type=int, help="ID de carta: traduce la carta y todos sus artículos (_carta_id).")
|
|
g.add_argument("--ids-file", help="Fichero con un ID de post ES por línea.")
|
|
ap.add_argument("--langs", default="en,fr,it,pt", help="Idiomas destino separados por coma.")
|
|
ap.add_argument("--status", default="draft", choices=["draft", "publish"], help="Estado de la traducción.")
|
|
ap.add_argument("--force", action="store_true", help="Regenera aunque ya exista la traducción.")
|
|
args = ap.parse_args()
|
|
|
|
langs = [l.strip() for l in args.langs.split(",") if l.strip() in LANG_NAMES]
|
|
if not langs:
|
|
log("Sin idiomas válidos."); return 1
|
|
|
|
if args.post_id:
|
|
ids = [args.post_id]
|
|
elif args.ids_file:
|
|
ids = [int(x) for x in Path(args.ids_file).read_text().split() if x.strip().isdigit()]
|
|
log(f"ids-file {args.ids_file}: {len(ids)} posts")
|
|
else:
|
|
ids = [args.carta] + carta_article_ids(args.carta)
|
|
log(f"Carta {args.carta}: {len(ids)} posts (carta + {len(ids)-1} artículos)")
|
|
|
|
state = load_state()
|
|
for pid in ids:
|
|
process_post(pid, langs, args.status, args.force, state)
|
|
save_state(state)
|
|
log(f"FIN. {len(state['done'])} traducciones registradas, "
|
|
f"{len(state.get('errors', {}))} errores. Estado: {STATE_FILE}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|