rafa
66ee943da4
El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
66 lines
3.3 KiB
Python
66 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
# Capa 1: sobre report.jsonl añade detección de idioma (lingua) y flags. Salida CSV+JSON. Coste 0.
|
|
import json, csv, os, sys, re
|
|
from lingua import Language, LanguageDetectorBuilder
|
|
DIR=os.path.dirname(os.path.abspath(__file__))
|
|
LANGMAP={'es':Language.SPANISH,'en':Language.ENGLISH,'fr':Language.FRENCH,'it':Language.ITALIAN,'pt':Language.PORTUGUESE}
|
|
det=LanguageDetectorBuilder.from_languages(*LANGMAP.values()).build()
|
|
code={Language.SPANISH:'es',Language.ENGLISH:'en',Language.FRENCH:'fr',Language.ITALIAN:'it',Language.PORTUGUESE:'pt'}
|
|
|
|
def detect(text):
|
|
t=(text or '').strip()
|
|
if len(t)<60: return ('',0.0)
|
|
l=det.detect_language_of(t)
|
|
if not l: return ('',0.0)
|
|
conf=det.compute_language_confidence(t,l)
|
|
return (code.get(l,''),round(conf,2))
|
|
|
|
rows=[]
|
|
with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f:
|
|
for line in f:
|
|
line=line.strip()
|
|
if not line: continue
|
|
r=json.loads(line)
|
|
flags=[]; exp=r.get('lang')
|
|
st=r.get('status',0)
|
|
if st==-1: flags.append('CRAWL_ERROR')
|
|
elif st>=500: flags.append('HTTP_5XX')
|
|
elif st>=400: flags.append('HTTP_4XX')
|
|
# redirección cross-idioma (solo content/menu; categorías de 1 carta redirigen legítimamente)
|
|
fl=r.get('final_lang','')
|
|
if r.get('redirected') and fl and exp and fl!=exp:
|
|
# ES esperado pero acaba en /xx/ o viceversa
|
|
flags.append('CROSS_LANG_REDIRECT')
|
|
dl,conf=detect(r.get('text',''))
|
|
r['detected_lang']=dl; r['detect_conf']=conf
|
|
if exp and exp!='es' and dl=='es' and conf>=0.7:
|
|
flags.append('SPANISH_LEAK')
|
|
if exp and dl and exp!=dl and 'SPANISH_LEAK' not in flags and conf>=0.85 and exp!='es':
|
|
flags.append('LANG_MISMATCH')
|
|
hl=(r.get('htmlLang') or '').split('-')[0].lower()
|
|
if hl and exp and hl!=exp and hl in LANGMAP:
|
|
flags.append('HTML_LANG_MISMATCH')
|
|
sev=0
|
|
if any(x in flags for x in ('HTTP_5XX','CRAWL_ERROR')): sev=3
|
|
elif any(x in flags for x in ('HTTP_4XX','CROSS_LANG_REDIRECT')): sev=2
|
|
elif any(x in flags for x in ('SPANISH_LEAK','LANG_MISMATCH','HTML_LANG_MISMATCH')): sev=1
|
|
r['flags']=flags; r['severity']=sev
|
|
rows.append(r)
|
|
|
|
rows.sort(key=lambda r:(-r['severity'], r['lang'], r['url']))
|
|
# JSON completo (sin el campo links para tamaño; se usa en link_audit aparte)
|
|
with open(os.path.join(DIR,'report.json'),'w',encoding='utf-8') as f:
|
|
json.dump([{k:v for k,v in r.items() if k not in('links','text','key')} for r in rows],f,ensure_ascii=False,indent=1)
|
|
# CSV
|
|
with open(os.path.join(DIR,'report.csv'),'w',newline='',encoding='utf-8') as f:
|
|
w=csv.writer(f); w.writerow(['sev','flags','lang','kind','status','detected','conf','redir','url','final'])
|
|
for r in rows:
|
|
if not r['flags'] and r.get('status')==200: continue # CSV solo lo accionable
|
|
w.writerow([r['severity'],'|'.join(r['flags']),r['lang'],r['kind'],r.get('status'),r.get('detected_lang'),r.get('detect_conf'),'Y' if r.get('redirected') else '',r['url'],r.get('final')])
|
|
# resumen
|
|
from collections import Counter
|
|
c=Counter(x for r in rows for x in r['flags'])
|
|
print('total analizadas:',len(rows))
|
|
print('flags:',dict(c))
|
|
print('accionables (en CSV):',sum(1 for r in rows if r['flags'] or r.get('status')!=200))
|