From 66ee943da4c48557ddb044def04d4946f1dd464e Mon Sep 17 00:00:00 2001 From: rafa Date: Mon, 22 Jun 2026 08:30:18 -0400 Subject: [PATCH] =?UTF-8?q?chore(verify):=20recuperar=20enum=5Furls.php=20?= =?UTF-8?q?y=20commitear=20tooling=20de=20verificaci=C3=B3n=20#121?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 --- tools/e2e/verify/.gitignore | 12 +++ tools/e2e/verify/README.md | 53 ++++++++++++ tools/e2e/verify/analyze.py | 65 +++++++++++++++ tools/e2e/verify/classify_gaps.py | 132 ++++++++++++++++++++++++++++++ tools/e2e/verify/crawl.cjs | 71 ++++++++++++++++ tools/e2e/verify/enum_urls.php | 88 ++++++++++++++++++++ tools/e2e/verify/link_audit.py | 70 ++++++++++++++++ tools/e2e/verify/llm_review.py | 48 +++++++++++ tools/e2e/verify/reverify.cjs | 30 +++++++ tools/e2e/verify/run_all.sh | 13 +++ 10 files changed, 582 insertions(+) create mode 100644 tools/e2e/verify/.gitignore create mode 100644 tools/e2e/verify/README.md create mode 100644 tools/e2e/verify/analyze.py create mode 100644 tools/e2e/verify/classify_gaps.py create mode 100644 tools/e2e/verify/crawl.cjs create mode 100644 tools/e2e/verify/enum_urls.php create mode 100644 tools/e2e/verify/link_audit.py create mode 100644 tools/e2e/verify/llm_review.py create mode 100644 tools/e2e/verify/reverify.cjs create mode 100755 tools/e2e/verify/run_all.sh diff --git a/tools/e2e/verify/.gitignore b/tools/e2e/verify/.gitignore new file mode 100644 index 0000000..afbb472 --- /dev/null +++ b/tools/e2e/verify/.gitignore @@ -0,0 +1,12 @@ +.venv/ +report.jsonl +report.json +report.csv +broken_links.json +llm_review.json +urls.json +gaps_dossier.json +gaps_traducir.csv +gaps_descargar.csv +run.log +prev-*/ diff --git a/tools/e2e/verify/README.md b/tools/e2e/verify/README.md new file mode 100644 index 0000000..d7cdd09 --- /dev/null +++ b/tools/e2e/verify/README.md @@ -0,0 +1,53 @@ +# fea-verify — Verificación automática del sitio (issue #121) + +Proceso por capas para validar el multiidioma del WP local **sin coste** (todo OSS + Gemma local). +Se ejecuta **contra el LOCAL** `https://farmer.taild3aaf6.ts.net/fea/` (prod está tras Cloudflare y bloquea headless). + +## Principio +Lo determinista (barato, exhaustivo) primero; el LLM solo sobre lo que las capas baratas marcan. + +## Capas / scripts +| Paso | Script | Qué hace | Coste | +|------|--------|----------|-------| +| Enumerar | `enum_urls.php` (en contenedor `wordpress-web`) | Saca de la BD todas las URLs: menús×5 idiomas, categorías con contenido, contenido no-ES. → `urls.json` | 0 | +| Capa 0+1 crawl | `crawl.cjs` (Playwright) | status HTTP, URL final (redirección cross-idioma), ``, texto visible, enlaces salientes. Resumible. → `report.jsonl` | 0 | +| Capa 1 análisis | `analyze.py` (lingua, venv) | detección de idioma offline + flags. → `report.json` + `report.csv` | 0 | +| Capa 0 enlaces | `link_audit.py` (stdlib) | comprueba status de los enlaces internos hallados. → `broken_links.json` | 0 | +| Capa 3 (opcional) | `llm_review.py` (Gemma local) | review semántica SOLO de páginas marcadas. → `llm_review.json` | 0 | + +## Requisitos (una vez) +- Node + Playwright: ya en `tools/e2e/node_modules`. +- venv con lingua: `python3 -m venv verify/.venv && verify/.venv/bin/pip install lingua-language-detector` +- Docker WP arriba (`wordpress-web`, `wordpress-mysql`), LM Studio con `google/gemma-4-e4b` (solo Capa 3). + +## Ejecutar (todo) +```bash +cd tools/e2e/verify +# 1) enumerar URLs desde la BD +docker cp enum_urls.php wordpress-web:/tmp/ && docker exec wordpress-web php /tmp/enum_urls.php && docker cp wordpress-web:/tmp/verify_urls.json urls.json +# 2) crawl (resumible; borra report.jsonl para empezar de cero) +CONC=5 node crawl.cjs +# 3) análisis + flags +.venv/bin/python analyze.py +# 4) enlaces rotos +python3 link_audit.py +# 5) (opcional) review LLM local sobre lo marcado +.venv/bin/python llm_review.py +``` + +## Leer el resultado +- **`report.csv`** — lo accionable, ordenado por gravedad. Columnas: `sev`(3 crítico→1 aviso), `flags`, `lang`, `kind`, `status`, `detected`(idioma detectado), `conf`, `redir`, `url`, `final`. +- **`broken_links.json`** — enlaces internos rotos + qué páginas los referencian. +- **`report.json`** — todo, en JSON. + +### Flags +- `HTTP_5XX` / `HTTP_4XX` / `CRAWL_ERROR` — la página falla (p.ej. el 500 de `/en/category/letters-from-other-weeks/`). +- `CROSS_LANG_REDIRECT` — pediste `/en/x/` y acabaste en otro idioma (bug de slug duplicado). +- `SPANISH_LEAK` — página no-ES cuyo texto detectado es español (sin traducir / widget en ES). +- `LANG_MISMATCH` / `HTML_LANG_MISMATCH` — idioma del texto / del `` ≠ esperado. + +## Scope de la corrida +Por defecto crawlea: menús (5 idiomas), categorías con contenido, y **todo el contenido no-ES** (~3.9k). El contenido ES masivo (~24k) se omite por volumen; añadirlo es cambiar la query de `enum_urls.php`. + +## Coste +**0 €.** Sin SaaS ni APIs de pago. Detección de idioma offline (lingua) y review con Gemma local. Minimax NO se usa. diff --git a/tools/e2e/verify/analyze.py b/tools/e2e/verify/analyze.py new file mode 100644 index 0000000..f55afce --- /dev/null +++ b/tools/e2e/verify/analyze.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Capa 1: sobre report.jsonl añade detección de idioma (lingua) y flags. Salida CSV+JSON. Coste 0. +import json, csv, os, sys, re +from lingua import Language, LanguageDetectorBuilder +DIR=os.path.dirname(os.path.abspath(__file__)) +LANGMAP={'es':Language.SPANISH,'en':Language.ENGLISH,'fr':Language.FRENCH,'it':Language.ITALIAN,'pt':Language.PORTUGUESE} +det=LanguageDetectorBuilder.from_languages(*LANGMAP.values()).build() +code={Language.SPANISH:'es',Language.ENGLISH:'en',Language.FRENCH:'fr',Language.ITALIAN:'it',Language.PORTUGUESE:'pt'} + +def detect(text): + t=(text or '').strip() + if len(t)<60: return ('',0.0) + l=det.detect_language_of(t) + if not l: return ('',0.0) + conf=det.compute_language_confidence(t,l) + return (code.get(l,''),round(conf,2)) + +rows=[] +with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f: + for line in f: + line=line.strip() + if not line: continue + r=json.loads(line) + flags=[]; exp=r.get('lang') + st=r.get('status',0) + if st==-1: flags.append('CRAWL_ERROR') + elif st>=500: flags.append('HTTP_5XX') + elif st>=400: flags.append('HTTP_4XX') + # redirección cross-idioma (solo content/menu; categorías de 1 carta redirigen legítimamente) + fl=r.get('final_lang','') + if r.get('redirected') and fl and exp and fl!=exp: + # ES esperado pero acaba en /xx/ o viceversa + flags.append('CROSS_LANG_REDIRECT') + dl,conf=detect(r.get('text','')) + r['detected_lang']=dl; r['detect_conf']=conf + if exp and exp!='es' and dl=='es' and conf>=0.7: + flags.append('SPANISH_LEAK') + if exp and dl and exp!=dl and 'SPANISH_LEAK' not in flags and conf>=0.85 and exp!='es': + flags.append('LANG_MISMATCH') + hl=(r.get('htmlLang') or '').split('-')[0].lower() + if hl and exp and hl!=exp and hl in LANGMAP: + flags.append('HTML_LANG_MISMATCH') + sev=0 + if any(x in flags for x in ('HTTP_5XX','CRAWL_ERROR')): sev=3 + elif any(x in flags for x in ('HTTP_4XX','CROSS_LANG_REDIRECT')): sev=2 + elif any(x in flags for x in ('SPANISH_LEAK','LANG_MISMATCH','HTML_LANG_MISMATCH')): sev=1 + r['flags']=flags; r['severity']=sev + rows.append(r) + +rows.sort(key=lambda r:(-r['severity'], r['lang'], r['url'])) +# JSON completo (sin el campo links para tamaño; se usa en link_audit aparte) +with open(os.path.join(DIR,'report.json'),'w',encoding='utf-8') as f: + json.dump([{k:v for k,v in r.items() if k not in('links','text','key')} for r in rows],f,ensure_ascii=False,indent=1) +# CSV +with open(os.path.join(DIR,'report.csv'),'w',newline='',encoding='utf-8') as f: + w=csv.writer(f); w.writerow(['sev','flags','lang','kind','status','detected','conf','redir','url','final']) + for r in rows: + if not r['flags'] and r.get('status')==200: continue # CSV solo lo accionable + w.writerow([r['severity'],'|'.join(r['flags']),r['lang'],r['kind'],r.get('status'),r.get('detected_lang'),r.get('detect_conf'),'Y' if r.get('redirected') else '',r['url'],r.get('final')]) +# resumen +from collections import Counter +c=Counter(x for r in rows for x in r['flags']) +print('total analizadas:',len(rows)) +print('flags:',dict(c)) +print('accionables (en CSV):',sum(1 for r in rows if r['flags'] or r.get('status')!=200)) diff --git a/tools/e2e/verify/classify_gaps.py b/tools/e2e/verify/classify_gaps.py new file mode 100644 index 0000000..9562cf0 --- /dev/null +++ b/tools/e2e/verify/classify_gaps.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +classify_gaps.py — Clasifica los huecos de traducción aflorados por el crawler +(#121/#132) en DESCARGAR (bíblico/litúrgico) vs TRADUCIR (comentarios/artículos), +para coordinar con Minimax. + +Entrada: report.csv (de analyze.py). Cruza las páginas marcadas SPANISH_LEAK / +LANG_MISMATCH / CROSS_LANG_REDIRECT con las categorías WP del post, vía wp eval. + +Salida: + gaps_traducir.csv — posts ES a traducir (comentarios/artículos) + gaps_descargar.csv — posts bíblicos/litúrgicos (lecturas, evangelios, salmos) + gaps_dossier.json — todo, estructurado +""" +import csv, json, subprocess, re, os, sys +from collections import defaultdict + +DIR = os.path.dirname(os.path.abspath(__file__)) +REPORT = os.path.join(DIR, "report.csv") + +# Categorías que se DESCARGAN (texto oficial), no se traducen con LLM. +DESCARGAR_CATS = { + 1645, # Lecturas bíblicas + 1648, # Eucaristía (lecturas/salmos litúrgicos) + 28, # evangelios-y-comentarios (los 4 evangelistas: descarga) +} +# term_id de las categorías que SÍ se traducen +TRADUCIR_CATS = {1646, 1647, 1650, 6} # editorial, comentarios evangelio, artículos, carta + +FLAGS_INTERES = ("SPANISH_LEAK", "LANG_MISMATCH", "HTML_LANG_MISMATCH", "CROSS_LANG_REDIRECT") + + +def slug_from_url(url): + path = re.sub(r"https?://[^/]+", "", url).strip("/") + parts = [p for p in path.split("/") if p] + if parts and parts[0] in ("en", "fr", "it", "pt", "es"): + parts = parts[1:] + # saltar prefijos no-post + if parts and parts[0] in ("category", "tag", "author", "page"): + return None + return parts[-1] if parts else None + + +def wp_resolve(slugs): + """Devuelve {slug: {id, lang, cats:[ids], title, es_id}} vía wp eval.""" + if not slugs: + return {} + payload = json.dumps(list(slugs)) + php = r''' +$slugs=json_decode(file_get_contents("php://stdin"),true); +$out=[]; +foreach($slugs as $s){ + $p=get_posts(["name"=>$s,"post_type"=>"post","post_status"=>["publish","draft"],"numberposts"=>1,"suppress_filters"=>true]); + if(!$p){$out[$s]=null;continue;} + $id=$p[0]->ID; + $lang=function_exists("pll_get_post_language")?pll_get_post_language($id):""; + $es=$id; + if($lang!=="es"&&function_exists("pll_get_post")){$t=pll_get_post($id,"es");if($t)$es=$t;} + $out[$s]=["id"=>$id,"lang"=>$lang,"cats"=>wp_get_post_categories($id),"title"=>get_the_title($id),"es_id"=>$es, + "es_cats"=>wp_get_post_categories($es),"es_title"=>get_the_title($es)]; +} +echo json_encode($out); +''' + r = subprocess.run( + ["docker", "exec", "-i", "wordpress-web", "wp", "eval", php, "--allow-root"], + input=payload, capture_output=True, text=True) + try: + return json.loads(r.stdout.strip().splitlines()[-1]) + except Exception as e: + print("wp_resolve error:", e, r.stdout[:300], r.stderr[:300], file=sys.stderr) + return {} + + +def main(): + if not os.path.exists(REPORT): + print("Falta report.csv — ejecuta analyze.py primero", file=sys.stderr) + return 1 + rows = list(csv.DictReader(open(REPORT, encoding="utf-8"))) + flagged = [r for r in rows if any(f in (r.get("flags") or "") for f in FLAGS_INTERES)] + print(f"Páginas marcadas: {len(flagged)} / {len(rows)}") + + slugs = {} + for r in flagged: + s = slug_from_url(r.get("url", "")) + if s: + slugs.setdefault(s, r) + + meta = wp_resolve(set(slugs)) + + traducir, descargar, otros = [], [], [] + for s, r in slugs.items(): + m = meta.get(s) + if not m: + otros.append({"slug": s, "url": r["url"], "flags": r.get("flags"), "motivo": "no resuelve post"}) + continue + cats = set(m.get("es_cats") or m.get("cats") or []) + row = {"slug": s, "lang": r.get("lang"), "flags": r.get("flags"), + "es_id": m["es_id"], "es_title": m.get("es_title"), "cats": sorted(cats), "url": r["url"]} + if cats & DESCARGAR_CATS: + descargar.append(row) + elif cats & TRADUCIR_CATS: + traducir.append(row) + else: + otros.append(row) + + # dedup por es_id para traducir (lo que va a Minimax) + by_es = {} + for t in traducir: + by_es.setdefault(t["es_id"], t) + + def write_csv(path, rows, cols): + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + w.writerows(rows) + + write_csv(os.path.join(DIR, "gaps_traducir.csv"), list(by_es.values()), + ["es_id", "es_title", "lang", "cats", "flags", "url"]) + write_csv(os.path.join(DIR, "gaps_descargar.csv"), descargar, + ["es_id", "es_title", "lang", "cats", "flags", "url"]) + json.dump({"traducir": list(by_es.values()), "descargar": descargar, "otros": otros}, + open(os.path.join(DIR, "gaps_dossier.json"), "w"), ensure_ascii=False, indent=2) + + print(f"\nTRADUCIR (Minimax): {len(by_es)} posts ES únicos") + print(f"DESCARGAR (bíblico/litúrgico): {len(descargar)} páginas") + print(f"OTROS (revisar a mano): {len(otros)}") + print("\nIDs ES a traducir:", ",".join(str(i) for i in by_es)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/e2e/verify/crawl.cjs b/tools/e2e/verify/crawl.cjs new file mode 100644 index 0000000..1320520 --- /dev/null +++ b/tools/e2e/verify/crawl.cjs @@ -0,0 +1,71 @@ +// Capa 0+1 (crawl): status, URL final (cross-lang), html lang, texto visible, enlaces salientes. +// Resumible: añade a report.jsonl; al rearrancar salta lo ya hecho. Coste 0 (local). +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +const DIR = __dirname; +const URLS = JSON.parse(fs.readFileSync(path.join(DIR,'urls.json'),'utf8')); +const OUT = path.join(DIR,'report.jsonl'); +const CONCURRENCY = parseInt(process.env.CONC||'3',10); +const LIMIT = parseInt(process.env.LIMIT||'0',10); // 0 = todas + +// resumir +const done = new Set(); +if (fs.existsSync(OUT)) for (const l of fs.readFileSync(OUT,'utf8').split('\n')) { + if(!l.trim()) continue; try { done.add(JSON.parse(l).key); } catch(e){} +} +let tasks = URLS.map(u=>({...u, key:u.lang+'|'+u.url})).filter(t=>!done.has(t.key)); +if (LIMIT) tasks = tasks.slice(0,LIMIT); +console.log(`pendientes: ${tasks.length} (ya hechas ${done.size})`); + +const langPrefix = (url) => { + const m = url.match(/\/fea\/(es|en|fr|it|pt)\//); if (m) return m[1]; + // sin prefijo bajo /fea/ = es (hide_default) + if (/\/fea\//.test(url)) return 'es'; + return ''; +}; + +(async () => { + const browser = await chromium.launch({ args:['--ignore-certificate-errors'] }); + const ctx = await browser.newContext({ ignoreHTTPSErrors:true, userAgent:'fea-verify-crawler' }); + const out = fs.createWriteStream(OUT, { flags:'a' }); + let i=0, done2=0; + const worker = async () => { + const page = await ctx.newPage(); + while (i < tasks.length) { + const t = tasks[i++]; + const rec = { key:t.key, url:t.url, lang:t.lang, kind:t.kind, id:t.id }; + let resp=null, lastErr=null; + for (let a=0; a<3 && !resp; a++) { + try { resp = await page.goto(t.url, { waitUntil:'domcontentloaded', timeout:30000 }); } + catch(e){ lastErr=e; await page.waitForTimeout(700); } + } + try { + if (!resp) throw lastErr || new Error('no response'); + rec.status = resp ? resp.status() : 0; + rec.final = page.url(); + rec.redirected = rec.final !== t.url; + rec.final_lang = langPrefix(rec.final); + const data = await page.evaluate(() => { + const main = document.querySelector('main, .wp-block-post-content, .entry-content, article') || document.body; + const text = (main.innerText||'').replace(/\s+/g,' ').trim().slice(0,1200); + const htmlLang = document.documentElement.getAttribute('lang')||''; + const links = Array.from(document.querySelectorAll('a[href]')).map(a=>a.href) + .filter(h=>/^https?:/.test(h)); + return { text, htmlLang, links:[...new Set(links)] }; + }); + rec.text = data.text; rec.htmlLang = data.htmlLang; rec.links = data.links; + } catch (e) { + rec.status = -1; rec.error = String(e).slice(0,120); rec.final=t.url; rec.links=[]; rec.text=''; + } + out.write(JSON.stringify(rec)+'\n'); + if (++done2 % 100 === 0) console.log(` ${done2}/${tasks.length}`); + } + await page.close(); + }; + await Promise.all(Array.from({length:CONCURRENCY}, worker)); + out.end(); + await browser.close(); + console.log(`crawl terminado: ${done2} nuevas, total report = ${done.size+done2}`); +})(); diff --git a/tools/e2e/verify/enum_urls.php b/tools/e2e/verify/enum_urls.php new file mode 100644 index 0000000..b920aca --- /dev/null +++ b/tools/e2e/verify/enum_urls.php @@ -0,0 +1,88 @@ + $url, 'lang' => $lang, 'kind' => $kind, 'id' => (int) $id]; +} + +// ── 1) MENÚ: 31 rutas base × 5 idiomas ─────────────────────────────────────── +// Rutas del menú principal (wp_navigation ID 1). Para no-ES se prefija /{lang}/; +// el crawler sigue redirecciones cross-idioma y las marca. +$menu_paths = [ + '', 'colaboradores', 'portal', 'paraponeraldialafe', 'category/cartasemana', + 'category/carta-semana-pasada', 'category/cartas-de-otras-semanas', + 'nueva-politica-de-privacidad', 'contactar', 'alta', 'alta-en-effa', 'escuela', + 'category/cartas-que-nos-llegan', 'category/tablon-de-anuncios', + 'category/asociacion-feadulta', 'numeros', 'category/comunidades-cristianas', + 'evangelio-diario-2024', 'category/indice-cronologico', + 'category/evangelios-y-comentarios', 'oraciones-eucaristicas', 'a-modo-de-salmo', + 'preces-y-oraciones-varias', 'te-creia-un-capricho-mas', 'autores-lista', 'temas', + 'multimedia', 'pensamientos', 'indice-cantoral', 'peliculas', 'in-memoriam', +]; +$home = home_url('/'); // .../fea/ +foreach ($LANGS as $lang) { + $prefix = ($lang === 'es') ? $home : $home . $lang . '/'; + foreach ($menu_paths as $p) { + add($out, $seen, $p === '' ? $prefix : $prefix . $p, $lang, 'menu', 0); + } +} + +// ── 2) CATEGORÍAS con contenido ────────────────────────────────────────────── +// ES: todas las categorías hide_empty. no-ES: solo las que tienen traducción. +$terms = get_terms(['taxonomy' => 'category', 'hide_empty' => true]); +foreach ($terms as $t) { + $link = get_term_link($t); + if (!is_wp_error($link)) add($out, $seen, $link, 'es', 'category', $t->term_id); + if (function_exists('pll_get_term')) { + foreach (['en', 'fr', 'it', 'pt'] as $lang) { + $tr = pll_get_term($t->term_id, $lang); + if ($tr) { + $l = get_term_link((int) $tr, 'category'); + if (!is_wp_error($l)) add($out, $seen, $l, $lang, 'category', $tr); + } + } + } +} + +// ── 3) CONTENIDO no-ES (posts publicados en en/fr/it/pt) ───────────────────── +foreach (['en', 'fr', 'it', 'pt'] as $lang) { + $q = new WP_Query([ + 'post_type' => 'post', + 'post_status' => 'publish', + 'posts_per_page' => -1, + 'fields' => 'ids', + 'lang' => $lang, + 'no_found_rows' => true, + ]); + foreach ($q->posts as $pid) { + add($out, $seen, get_permalink($pid), $lang, 'content', $pid); + } +} + +file_put_contents('/tmp/verify_urls.json', json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); +$by = []; +foreach ($out as $r) { $by[$r['kind']] = ($by[$r['kind']] ?? 0) + 1; } +fwrite(STDERR, 'enum: ' . count($out) . ' URLs ' . json_encode($by) . "\n"); diff --git a/tools/e2e/verify/link_audit.py b/tools/e2e/verify/link_audit.py new file mode 100644 index 0000000..df3beb0 --- /dev/null +++ b/tools/e2e/verify/link_audit.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Capa 0 (enlaces): comprueba status de los enlaces internos salientes hallados en el crawl. +# Marca rotos (>=400/error) y qué páginas los referencian. stdlib, coste 0. +import json, os, ssl, urllib.request, urllib.parse +from concurrent.futures import ThreadPoolExecutor, as_completed +DIR=os.path.dirname(os.path.abspath(__file__)) +ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE +SITE='farmer.taild3aaf6.ts.net' +CAP=int(os.environ.get('CAP','9000')) + +def norm(u): + return u.split('#')[0].rstrip('/') + +crawled={} # url normalizada -> status (de report) +referers={} # link -> set(paginas) +links=set() +with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f: + for line in f: + line=line.strip() + if not line: continue + r=json.loads(line) + if r.get('final'): crawled[norm(r['final'])]=r.get('status') + crawled.setdefault(norm(r['url']), r.get('status')) + for l in (r.get('links') or []): + if SITE in l: + n=norm(l); links.add(n); referers.setdefault(n,set()).add(r['url']) + +# solo comprobar las que NO conocemos ya por el crawl +todo=[l for l in links if l not in crawled] +capped = len(todo)>CAP +todo=todo[:CAP] +print(f"enlaces internos únicos: {len(links)} | ya conocidos: {len(links)-len(todo)-(len(todo) if False else 0)} | a comprobar: {len(todo)}"+(" (CAP)" if capped else "")) + +def check(u): + try: + req=urllib.request.Request(u, method='HEAD', headers={'User-Agent':'fea-link-audit'}) + with urllib.request.urlopen(req,timeout=15,context=ctx) as r: return (u,r.status) + except urllib.error.HTTPError as e: + if e.code in (405,501): # HEAD no permitido → GET + try: + req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'}) + with urllib.request.urlopen(req,timeout=20,context=ctx) as r: return (u,r.status) + except Exception as e2: return (u, getattr(e2,'code',-1)) + return (u,e.code) + except Exception: + # reintento único (evita falsos -1 por saturación) + try: + req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'}) + with urllib.request.urlopen(req,timeout=25,context=ctx) as r: return (u,r.status) + except Exception: return (u,-1) + +res={} +with ThreadPoolExecutor(max_workers=8) as ex: + futs=[ex.submit(check,u) for u in todo] + for i,fu in enumerate(as_completed(futs)): + u,s=fu.result(); res[u]=s + if (i+1)%500==0: print(f" {i+1}/{len(todo)}") + +# combinar +allstatus=dict(crawled); allstatus.update(res) +broken=[] +for l in links: + s=allstatus.get(l) + if s is not None and (s==-1 or s>=400): + broken.append({'link':l,'status':s,'referenced_by':sorted(referers.get(l,[]))[:8],'n_referers':len(referers.get(l,[]))}) +broken.sort(key=lambda b:(-b['n_referers'], b['link'])) +json.dump({'capped':capped,'checked':len(todo),'unique_links':len(links),'broken':broken}, + open(os.path.join(DIR,'broken_links.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1) +print(f"ENLACES ROTOS: {len(broken)} (de {len(links)} únicos)") +for b in broken[:15]: print(f" [{b['status']}] {b['link']} <- {b['n_referers']} págs") diff --git a/tools/e2e/verify/llm_review.py b/tools/e2e/verify/llm_review.py new file mode 100644 index 0000000..480d2f9 --- /dev/null +++ b/tools/e2e/verify/llm_review.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Capa 3 (opcional): review semántica con Gemma LOCAL, SOLO sobre páginas marcadas por la Capa 1. +# Coste 0 (LLM local). Si Gemma no responde, sale limpio y deja constancia. +import json, os, urllib.request +DIR=os.path.dirname(os.path.abspath(__file__)) +GEMMA="http://172.19.128.1:1234/v1/chat/completions"; MODEL="google/gemma-4-e4b" +MAX=int(os.environ.get('MAX','40')) +LNAME={'en':'inglés','fr':'francés','it':'italiano','pt':'portugués','es':'español'} + +# texto visible está en report.jsonl (no en report.json) +texts={} +for line in open(os.path.join(DIR,'report.jsonl'),encoding='utf-8'): + line=line.strip() + if not line: continue + r=json.loads(line); texts[(r['lang'],r['url'])]=r.get('text','') + +rep=json.load(open(os.path.join(DIR,'report.json'),encoding='utf-8')) +flagged=[r for r in rep if any(f in r.get('flags',[]) for f in ('SPANISH_LEAK','LANG_MISMATCH'))][:MAX] +print(f"páginas marcadas para review LLM: {len(flagged)} (cap {MAX})") + +def ask(lang,text): + prompt=(f"Eres revisor de QA de un sitio web. Esta página DEBERÍA estar en {LNAME.get(lang,lang)}. " + f"Evalúa el texto visible. Responde SOLO un JSON válido: " + f'{{"idioma_ok": true/false, "idioma_predominante": "es/en/fr/it/pt", "problemas": ["lista breve de mistraducciones, trozos sin traducir o sin sentido"]}}.\n\nTEXTO:\n'+text[:1500]) + body=json.dumps({"model":MODEL,"messages":[{"role":"user","content":prompt}],"temperature":0.1,"max_tokens":2200}).encode() + req=urllib.request.Request(GEMMA,data=body,headers={"Content-Type":"application/json"}) + r=json.load(urllib.request.urlopen(req,timeout=120)) + out=r["choices"][0]["message"]["content"].strip() + s=out.find('{'); e=out.rfind('}') + return json.loads(out[s:e+1]) if s>=0 else {"raw":out[:200]} + +results=[] +try: + urllib.request.urlopen("http://172.19.128.1:1234/v1/models",timeout=6) +except Exception as e: + print("Gemma no responde, salto Capa 3:",e); + json.dump({"skipped":True,"reason":str(e)},open(os.path.join(DIR,'llm_review.json'),'w')); raise SystemExit(0) + +for i,r in enumerate(flagged): + t=texts.get((r['lang'],r['url']),'') + if len(t)<60: continue + try: + v=ask(r['lang'],t); v.update({'url':r['url'],'lang':r['lang'],'flags':r['flags']}); results.append(v) + print(f" [{i+1}/{len(flagged)}] {r['lang']} ok={v.get('idioma_ok')} pred={v.get('idioma_predominante')}") + except Exception as e: + print(f" [{i+1}] error {e}") +json.dump(results,open(os.path.join(DIR,'llm_review.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1) +print(f"review LLM guardada: {len(results)} páginas") diff --git a/tools/e2e/verify/reverify.cjs b/tools/e2e/verify/reverify.cjs new file mode 100644 index 0000000..c694ba6 --- /dev/null +++ b/tools/e2e/verify/reverify.cjs @@ -0,0 +1,30 @@ +// Re-verifica SOLO los registros con status -1 (artefactos de carga), secuencial + reintentos. +const { chromium } = require('playwright'); +const fs=require('fs'), path=require('path'); +const DIR=__dirname, F=path.join(DIR,'report.jsonl'); +const recs=fs.readFileSync(F,'utf8').split('\n').filter(l=>l.trim()).map(l=>JSON.parse(l)); +const bad=recs.filter(r=>r.status===-1); +console.log(`a re-verificar: ${bad.length}`); +const langPrefix=(url)=>{const m=url.match(/\/fea\/(es|en|fr|it|pt)\//);if(m)return m[1];if(/\/fea\//.test(url))return 'es';return '';}; +(async()=>{ + const b=await chromium.launch({args:['--ignore-certificate-errors']}); + const ctx=await b.newContext({ignoreHTTPSErrors:true,userAgent:'fea-verify-reverify'}); + const page=await ctx.newPage(); + let fixed=0,stillbad=0; + for(let i=0;i{const m=document.querySelector('main,.wp-block-post-content,.entry-content,article')||document.body;return{text:(m.innerText||'').replace(/\s+/g,' ').trim().slice(0,1200),htmlLang:document.documentElement.getAttribute('lang')||'',links:[...new Set(Array.from(document.querySelectorAll('a[href]')).map(a=>a.href).filter(h=>/^https?:/.test(h)))]};}); + r.text=d.text;r.htmlLang=d.htmlLang;r.links=d.links;delete r.error; ok=true; fixed++; + }catch(e){ r.error=String(e).slice(0,120); await page.waitForTimeout(800); } + } + if(!ok) stillbad++; + if((i+1)%100===0) console.log(` ${i+1}/${bad.length} (recuperadas ${fixed}, aún mal ${stillbad})`); + } + await b.close(); + fs.writeFileSync(F, recs.map(r=>JSON.stringify(r)).join('\n')+'\n'); + console.log(`re-verify done: recuperadas ${fixed}, siguen mal ${stillbad}`); +})(); diff --git a/tools/e2e/verify/run_all.sh b/tools/e2e/verify/run_all.sh new file mode 100755 index 0000000..fcd7922 --- /dev/null +++ b/tools/e2e/verify/run_all.sh @@ -0,0 +1,13 @@ +#!/bin/bash +cd /home/rafa/joomla-migration/tools/e2e/verify +exec > run.log 2>&1 +echo "=== START $(date -u +%FT%TZ) ===" +echo "urls: $(python3 -c 'import json;print(len(json.load(open("urls.json"))))')" +echo "--- crawl (CONC=3) ---" +CONC=3 node crawl.cjs +echo "crawl lines: $(wc -l < report.jsonl)" +echo "--- analyze ---" +.venv/bin/python analyze.py +echo "--- link_audit ---" +python3 link_audit.py +echo "=== DONE $(date -u +%FT%TZ) ==="