chore(verify): recuperar enum_urls.php y commitear tooling de verificación #121

El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-22 08:30:18 -04:00
parent 863a3c6677
commit 66ee943da4
10 changed files with 582 additions and 0 deletions
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# Capa 1: sobre report.jsonl añade detección de idioma (lingua) y flags. Salida CSV+JSON. Coste 0.
+import json, csv, os, sys, re
+from lingua import Language, LanguageDetectorBuilder
+DIR=os.path.dirname(os.path.abspath(__file__))
+LANGMAP={'es':Language.SPANISH,'en':Language.ENGLISH,'fr':Language.FRENCH,'it':Language.ITALIAN,'pt':Language.PORTUGUESE}
+det=LanguageDetectorBuilder.from_languages(*LANGMAP.values()).build()
+code={Language.SPANISH:'es',Language.ENGLISH:'en',Language.FRENCH:'fr',Language.ITALIAN:'it',Language.PORTUGUESE:'pt'}
+
+def detect(text):
+    t=(text or '').strip()
+    if len(t)<60: return ('',0.0)
+    l=det.detect_language_of(t)
+    if not l: return ('',0.0)
+    conf=det.compute_language_confidence(t,l)
+    return (code.get(l,''),round(conf,2))
+
+rows=[]
+with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f:
+    for line in f:
+        line=line.strip()
+        if not line: continue
+        r=json.loads(line)
+        flags=[]; exp=r.get('lang')
+        st=r.get('status',0)
+        if st==-1: flags.append('CRAWL_ERROR')
+        elif st>=500: flags.append('HTTP_5XX')
+        elif st>=400: flags.append('HTTP_4XX')
+        # redirección cross-idioma (solo content/menu; categorías de 1 carta redirigen legítimamente)
+        fl=r.get('final_lang','')
+        if r.get('redirected') and fl and exp and fl!=exp:
+            # ES esperado pero acaba en /xx/ o viceversa
+            flags.append('CROSS_LANG_REDIRECT')
+        dl,conf=detect(r.get('text',''))
+        r['detected_lang']=dl; r['detect_conf']=conf
+        if exp and exp!='es' and dl=='es' and conf>=0.7:
+            flags.append('SPANISH_LEAK')
+        if exp and dl and exp!=dl and 'SPANISH_LEAK' not in flags and conf>=0.85 and exp!='es':
+            flags.append('LANG_MISMATCH')
+        hl=(r.get('htmlLang') or '').split('-')[0].lower()
+        if hl and exp and hl!=exp and hl in LANGMAP:
+            flags.append('HTML_LANG_MISMATCH')
+        sev=0
+        if any(x in flags for x in ('HTTP_5XX','CRAWL_ERROR')): sev=3
+        elif any(x in flags for x in ('HTTP_4XX','CROSS_LANG_REDIRECT')): sev=2
+        elif any(x in flags for x in ('SPANISH_LEAK','LANG_MISMATCH','HTML_LANG_MISMATCH')): sev=1
+        r['flags']=flags; r['severity']=sev
+        rows.append(r)
+
+rows.sort(key=lambda r:(-r['severity'], r['lang'], r['url']))
+# JSON completo (sin el campo links para tamaño; se usa en link_audit aparte)
+with open(os.path.join(DIR,'report.json'),'w',encoding='utf-8') as f:
+    json.dump([{k:v for k,v in r.items() if k not in('links','text','key')} for r in rows],f,ensure_ascii=False,indent=1)
+# CSV
+with open(os.path.join(DIR,'report.csv'),'w',newline='',encoding='utf-8') as f:
+    w=csv.writer(f); w.writerow(['sev','flags','lang','kind','status','detected','conf','redir','url','final'])
+    for r in rows:
+        if not r['flags'] and r.get('status')==200: continue  # CSV solo lo accionable
+        w.writerow([r['severity'],'|'.join(r['flags']),r['lang'],r['kind'],r.get('status'),r.get('detected_lang'),r.get('detect_conf'),'Y' if r.get('redirected') else '',r['url'],r.get('final')])
+# resumen
+from collections import Counter
+c=Counter(x for r in rows for x in r['flags'])
+print('total analizadas:',len(rows))
+print('flags:',dict(c))
+print('accionables (en CSV):',sum(1 for r in rows if r['flags'] or r.get('status')!=200))