rafa
66ee943da4
El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
71 lines
3.2 KiB
Python
71 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
# Capa 0 (enlaces): comprueba status de los enlaces internos salientes hallados en el crawl.
|
|
# Marca rotos (>=400/error) y qué páginas los referencian. stdlib, coste 0.
|
|
import json, os, ssl, urllib.request, urllib.parse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
DIR=os.path.dirname(os.path.abspath(__file__))
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
SITE='farmer.taild3aaf6.ts.net'
|
|
CAP=int(os.environ.get('CAP','9000'))
|
|
|
|
def norm(u):
|
|
return u.split('#')[0].rstrip('/')
|
|
|
|
crawled={} # url normalizada -> status (de report)
|
|
referers={} # link -> set(paginas)
|
|
links=set()
|
|
with open(os.path.join(DIR,'report.jsonl'),encoding='utf-8') as f:
|
|
for line in f:
|
|
line=line.strip()
|
|
if not line: continue
|
|
r=json.loads(line)
|
|
if r.get('final'): crawled[norm(r['final'])]=r.get('status')
|
|
crawled.setdefault(norm(r['url']), r.get('status'))
|
|
for l in (r.get('links') or []):
|
|
if SITE in l:
|
|
n=norm(l); links.add(n); referers.setdefault(n,set()).add(r['url'])
|
|
|
|
# solo comprobar las que NO conocemos ya por el crawl
|
|
todo=[l for l in links if l not in crawled]
|
|
capped = len(todo)>CAP
|
|
todo=todo[:CAP]
|
|
print(f"enlaces internos únicos: {len(links)} | ya conocidos: {len(links)-len(todo)-(len(todo) if False else 0)} | a comprobar: {len(todo)}"+(" (CAP)" if capped else ""))
|
|
|
|
def check(u):
|
|
try:
|
|
req=urllib.request.Request(u, method='HEAD', headers={'User-Agent':'fea-link-audit'})
|
|
with urllib.request.urlopen(req,timeout=15,context=ctx) as r: return (u,r.status)
|
|
except urllib.error.HTTPError as e:
|
|
if e.code in (405,501): # HEAD no permitido → GET
|
|
try:
|
|
req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'})
|
|
with urllib.request.urlopen(req,timeout=20,context=ctx) as r: return (u,r.status)
|
|
except Exception as e2: return (u, getattr(e2,'code',-1))
|
|
return (u,e.code)
|
|
except Exception:
|
|
# reintento único (evita falsos -1 por saturación)
|
|
try:
|
|
req=urllib.request.Request(u, headers={'User-Agent':'fea-link-audit'})
|
|
with urllib.request.urlopen(req,timeout=25,context=ctx) as r: return (u,r.status)
|
|
except Exception: return (u,-1)
|
|
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
futs=[ex.submit(check,u) for u in todo]
|
|
for i,fu in enumerate(as_completed(futs)):
|
|
u,s=fu.result(); res[u]=s
|
|
if (i+1)%500==0: print(f" {i+1}/{len(todo)}")
|
|
|
|
# combinar
|
|
allstatus=dict(crawled); allstatus.update(res)
|
|
broken=[]
|
|
for l in links:
|
|
s=allstatus.get(l)
|
|
if s is not None and (s==-1 or s>=400):
|
|
broken.append({'link':l,'status':s,'referenced_by':sorted(referers.get(l,[]))[:8],'n_referers':len(referers.get(l,[]))})
|
|
broken.sort(key=lambda b:(-b['n_referers'], b['link']))
|
|
json.dump({'capped':capped,'checked':len(todo),'unique_links':len(links),'broken':broken},
|
|
open(os.path.join(DIR,'broken_links.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
print(f"ENLACES ROTOS: {len(broken)} (de {len(links)} únicos)")
|
|
for b in broken[:15]: print(f" [{b['status']}] {b['link']} <- {b['n_referers']} págs")
|