#!/usr/bin/env python3
# Capa 0 (enlaces): comprueba status de los enlaces internos salientes hallados en el crawl.
# Marca rotos (>=400/error) y qué páginas los referencian. stdlib, coste 0.
import json, os, ssl, urllib.request, urllib.parse
from concurrent.futures import ThreadPoolExecutor, as_completed
DIR=os.path.dirname(os.path.abspath(__file__))
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
SITE='farmer.taild3aaf6.ts.net'
CAP=int(os.environ.get('CAP','9000'))

def norm(u):
    return u.split('#')[0].rstrip('/')

crawled={}   # url normalizada -> status (de report)
referers={}  # link -> set(paginas)
links=set()
with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f:
    for line in f:
        line=line.strip()
        if not line: continue
        r=json.loads(line)
        if r.get('final'): crawled[norm(r['final'])]=r.get('status')
        crawled.setdefault(norm(r['url']), r.get('status'))
        for l in (r.get('links') or []):
            if SITE in l:
                n=norm(l); links.add(n); referers.setdefault(n,set()).add(r['url'])

# solo comprobar las que NO conocemos ya por el crawl
todo=[l for l in links if l not in crawled]
capped = len(todo)>CAP
todo=todo[:CAP]
print(f"enlaces internos únicos: {len(links)} | ya conocidos: {len(links)-len(todo)-(len(todo) if False else 0)} | a comprobar: {len(todo)}"+(" (CAP)" if capped else ""))

def check(u):
    try:
        req=urllib.request.Request(u, method='HEAD', headers={'User-Agent':'fea-link-audit'})
        with urllib.request.urlopen(req,timeout=15,context=ctx) as r: return (u,r.status)
    except urllib.error.HTTPError as e:
        if e.code in (405,501):  # HEAD no permitido → GET
            try:
                req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'})
                with urllib.request.urlopen(req,timeout=20,context=ctx) as r: return (u,r.status)
            except Exception as e2: return (u, getattr(e2,'code',-1))
        return (u,e.code)
    except Exception:
        # reintento único (evita falsos -1 por saturación)
        try:
            req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'})
            with urllib.request.urlopen(req,timeout=25,context=ctx) as r: return (u,r.status)
        except Exception: return (u,-1)

res={}
with ThreadPoolExecutor(max_workers=8) as ex:
    futs=[ex.submit(check,u) for u in todo]
    for i,fu in enumerate(as_completed(futs)):
        u,s=fu.result(); res[u]=s
        if (i+1)%500==0: print(f"  {i+1}/{len(todo)}")

# combinar
allstatus=dict(crawled); allstatus.update(res)
broken=[]
for l in links:
    s=allstatus.get(l)
    if s is not None and (s==-1 or s>=400):
        broken.append({'link':l,'status':s,'referenced_by':sorted(referers.get(l,[]))[:8],'n_referers':len(referers.get(l,[]))})
broken.sort(key=lambda b:(-b['n_referers'], b['link']))
json.dump({'capped':capped,'checked':len(todo),'unique_links':len(links),'broken':broken},
          open(os.path.join(DIR,'broken_links.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1)
print(f"ENLACES ROTOS: {len(broken)} (de {len(links)} únicos)")
for b in broken[:15]: print(f"  [{b['status']}] {b['link']}  <- {b['n_referers']} págs")