#!/usr/bin/env python3
# Capa 1: sobre report.jsonl añade detección de idioma (lingua) y flags. Salida CSV+JSON. Coste 0.
import json, csv, os, sys, re
from lingua import Language, LanguageDetectorBuilder
DIR=os.path.dirname(os.path.abspath(__file__))
LANGMAP={'es':Language.SPANISH,'en':Language.ENGLISH,'fr':Language.FRENCH,'it':Language.ITALIAN,'pt':Language.PORTUGUESE}
det=LanguageDetectorBuilder.from_languages(*LANGMAP.values()).build()
code={Language.SPANISH:'es',Language.ENGLISH:'en',Language.FRENCH:'fr',Language.ITALIAN:'it',Language.PORTUGUESE:'pt'}

def detect(text):
    t=(text or '').strip()
    if len(t)<60: return ('',0.0)
    l=det.detect_language_of(t)
    if not l: return ('',0.0)
    conf=det.compute_language_confidence(t,l)
    return (code.get(l,''),round(conf,2))

rows=[]
with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f:
    for line in f:
        line=line.strip()
        if not line: continue
        r=json.loads(line)
        flags=[]; exp=r.get('lang')
        st=r.get('status',0)
        if st==-1: flags.append('CRAWL_ERROR')
        elif st>=500: flags.append('HTTP_5XX')
        elif st>=400: flags.append('HTTP_4XX')
        # redirección cross-idioma (solo content/menu; categorías de 1 carta redirigen legítimamente)
        fl=r.get('final_lang','')
        if r.get('redirected') and fl and exp and fl!=exp:
            # ES esperado pero acaba en /xx/ o viceversa
            flags.append('CROSS_LANG_REDIRECT')
        dl,conf=detect(r.get('text',''))
        r['detected_lang']=dl; r['detect_conf']=conf
        if exp and exp!='es' and dl=='es' and conf>=0.7:
            flags.append('SPANISH_LEAK')
        if exp and dl and exp!=dl and 'SPANISH_LEAK' not in flags and conf>=0.85 and exp!='es':
            flags.append('LANG_MISMATCH')
        hl=(r.get('htmlLang') or '').split('-')[0].lower()
        if hl and exp and hl!=exp and hl in LANGMAP:
            flags.append('HTML_LANG_MISMATCH')
        sev=0
        if any(x in flags for x in ('HTTP_5XX','CRAWL_ERROR')): sev=3
        elif any(x in flags for x in ('HTTP_4XX','CROSS_LANG_REDIRECT')): sev=2
        elif any(x in flags for x in ('SPANISH_LEAK','LANG_MISMATCH','HTML_LANG_MISMATCH')): sev=1
        r['flags']=flags; r['severity']=sev
        rows.append(r)

rows.sort(key=lambda r:(-r['severity'], r['lang'], r['url']))
# JSON completo (sin el campo links para tamaño; se usa en link_audit aparte)
with open(os.path.join(DIR,'report.json'),'w',encoding='utf-8') as f:
    json.dump([{k:v for k,v in r.items() if k not in('links','text','key')} for r in rows],f,ensure_ascii=False,indent=1)
# CSV
with open(os.path.join(DIR,'report.csv'),'w',newline='',encoding='utf-8') as f:
    w=csv.writer(f); w.writerow(['sev','flags','lang','kind','status','detected','conf','redir','url','final'])
    for r in rows:
        if not r['flags'] and r.get('status')==200: continue  # CSV solo lo accionable
        w.writerow([r['severity'],'|'.join(r['flags']),r['lang'],r['kind'],r.get('status'),r.get('detected_lang'),r.get('detect_conf'),'Y' if r.get('redirected') else '',r['url'],r.get('final')])
# resumen
from collections import Counter
c=Counter(x for r in rows for x in r['flags'])
print('total analizadas:',len(rows))
print('flags:',dict(c))
print('accionables (en CSV):',sum(1 for r in rows if r['flags'] or r.get('status')!=200))