#!/usr/bin/env python3
"""Clasifica missing-pages-sinquery.txt: separa lo que es ruido/codificacion de los huecos reales."""
import os, re
from collections import Counter
from urllib.parse import unquote, urlsplit

BASE = "/home/rafa/joomla-migration/mirror-antiguo"
RUN = open(os.path.join(BASE, "CURRENT_RUN")).read().strip()
DIR = os.path.join(BASE, "runs", RUN)
RAW = os.path.join(DIR, "raw", "antiguo.feadulta.com")

inv = set(l.strip() for l in open(os.path.join(BASE, "inventory", "urls-input.txt")))
inv_dec = set(unquote(u) for u in inv)

cats = Counter()
real = []
for line in open(os.path.join(DIR, "missing-pages-sinquery.txt")):
    u = line.strip()
    d = unquote(u)
    if d in inv_dec:
        cats["ya_en_inventario (solo difiere la codificacion %XX)"] += 1
        continue
    # ¿existe ya el fichero en disco?
    p = urlsplit(d).path
    fp = os.path.join(RAW, p.lstrip("/"))
    if p.endswith("/"):
        fp = os.path.join(fp, "index.html")
    if os.path.exists(fp):
        cats["ya_capturado en disco"] += 1
        continue

    if "/itemlist/user/" in d:
        cats["K2 pagina de autor (itemlist/user)"] += 1; real.append(u)
    elif "/itemlist/tag/" in d:
        cats["K2 pagina de etiqueta (itemlist/tag)"] += 1; real.append(u)
    elif "/itemlist/date/" in d or "/itemlist/category" in d:
        cats["K2 listado (fecha/categoria)"] += 1; real.append(u)
    elif d.startswith("http://antiguo.feadulta.com/anterior/"):
        cats["/anterior (web estatica antigua)"] += 1; real.append(u)
    elif d.startswith("http://antiguo.feadulta.com/ediciones/"):
        cats["/ediciones"] += 1; real.append(u)
    elif "/index.php/" in d:
        cats["enlace no-SEF (index.php/...)"] += 1; real.append(u)
    elif re.search(r"/ES/|/BUSCADORAVANZADO/", d):
        cats["enlace roto por mayusculas"] += 1
    else:
        cats["OTROS - revisar"] += 1; real.append(u)

for k, v in cats.most_common():
    print("%7d  %s" % (v, k))
print()
out = os.path.join(DIR, "missing-real.txt")
with open(out, "w") as f:
    for u in sorted(set(real)):
        f.write(u + "\n")
print("candidatos reales ->", out, len(set(real)))

print("\n--- muestra de OTROS ---")
n = 0
for u in sorted(set(real)):
    d = unquote(u)
    if not any(s in d for s in ("/itemlist/", "/anterior/", "/ediciones/", "/index.php/")):
        print("  ", u); n += 1
        if n >= 20: break