#!/usr/bin/env python3
"""Pase B (1/2): extrae de las paginas capturadas los enlaces internos.

Salidas en el directorio de la corrida:
  assets-input.txt       URLs internas a recursos NO html (css, js, img, pdf, mp3, doc...)
  missing-pages.txt      paginas .html internas enlazadas que NO estan en el inventario
  external-hosts.txt     hosts externos referenciados, con recuento
"""
import os, re, sys, html
from collections import Counter
from urllib.parse import urljoin, urlsplit, urlunsplit

BASE = "/home/rafa/joomla-migration/mirror-antiguo"
RUN = open(os.path.join(BASE, "CURRENT_RUN")).read().strip()
DIR = os.path.join(BASE, "runs", RUN)
RAW = os.path.join(DIR, "raw", "antiguo.feadulta.com")
HOST = "antiguo.feadulta.com"

ATTR = re.compile(rb'(?:href|src|data-src|poster)\s*=\s*["\']([^"\'>]+)["\']', re.I)
CSSURL = re.compile(rb'url\(\s*["\']?([^"\')]+)["\']?\s*\)', re.I)

SKIP_SCHEMES = ("mailto:", "javascript:", "tel:", "data:", "#", "skype:", "whatsapp:")

inventory = set()
for line in open(os.path.join(BASE, "inventory", "urls-input.txt")):
    inventory.add(line.strip())

assets, pages, ext = set(), set(), Counter()
nfiles = 0

def norm(u, page_url):
    u = html.unescape(u.strip())
    if not u or u.startswith(SKIP_SCHEMES):
        return None
    absu = urljoin(page_url, u)
    p = urlsplit(absu)
    if p.scheme not in ("http", "https"):
        return None
    if p.netloc.split(":")[0] != HOST:
        ext[p.netloc] += 1
        return None
    # sin fragmento; conservamos query (rara en assets)
    return urlunsplit(("http", HOST, p.path, p.query, ""))

for root, _dirs, files in os.walk(RAW):
    for fn in files:
        path = os.path.join(root, fn)
        rel = os.path.relpath(path, RAW)
        page_url = "http://%s/%s" % (HOST, rel.replace(os.sep, "/"))
        if not fn.lower().endswith((".html", ".htm")):
            continue
        nfiles += 1
        try:
            data = open(path, "rb").read()
        except OSError:
            continue
        found = ATTR.findall(data) + CSSURL.findall(data)
        for raw in found:
            try:
                u = norm(raw.decode("utf-8", "replace"), page_url)
            except ValueError:
                continue
            if not u:
                continue
            tail = urlsplit(u).path.lower()
            if tail.endswith((".html", ".htm")) or tail.endswith("/"):
                if u not in inventory:
                    pages.add(u)
            else:
                assets.add(u)

def dump(name, it):
    p = os.path.join(DIR, name)
    with open(p, "w") as f:
        for x in sorted(it):
            f.write(x + "\n")
    return p, len(it)

print("paginas HTML analizadas:", nfiles)
for n, c in (dump("assets-input.txt", assets), dump("missing-pages.txt", pages)):
    print(n, c)
with open(os.path.join(DIR, "external-hosts.txt"), "w") as f:
    for h, c in ext.most_common():
        f.write("%7d  %s\n" % (c, h))
print("hosts externos distintos:", len(ext))

# reparto de assets por extension
c = Counter()
for u in assets:
    e = os.path.splitext(urlsplit(u).path)[1].lower() or "(sin ext)"
    c[e] += 1
print("--- assets por extension ---")
for e, n in c.most_common(25):
    print("%7d  %s" % (n, e))