#!/bin/bash
# Pase C unificado: conjunto completo de URLs deseadas menos lo que ya esta en disco.
#   inventario  +  rutas de menu por `path`  +  huecos detectados en el HTML  +  assets reales
# wget con --no-clobber se salta lo ya descargado, asi que el script es idempotente.
set -uo pipefail
BASE=/home/rafa/joomla-migration/mirror-antiguo
RUN=$(cat "$BASE/CURRENT_RUN")
D=$BASE/runs/$RUN
W=${WORKERS:-4}
REJECT='(\?|&)(start|limitstart|limit|print|tmpl|format|searchword|task|orderby|filter|catid|month|year)='

cat "$BASE/inventory/urls-input.txt" \
    "$BASE/inventory/urls-menu-path.txt" \
    "$D/missing-real.txt" \
    <(grep -v 'antiguo.feadulta.com/es/' "$D/assets-input.txt") \
  | grep -E '^http://antiguo\.feadulta\.com/' \
  | grep -v 'print=1' \
  | grep -v 'tmpl=component' \
  | grep -v 'format=opensearch' \
  | grep -v '/component/mailto/' \
  | sort -u > "$D/passC-input.txt"

echo "conjunto deseado: $(wc -l < "$D/passC-input.txt")"
echo "ficheros en disco antes: $(find "$D/raw" -type f | wc -l)"

mkdir -p "$D/chunks-c"; rm -f "$D/chunks-c"/*.txt
split -n l/$W -d --additional-suffix=.txt "$D/passC-input.txt" "$D/chunks-c/c-"

date -u +%FT%TZ > "$D/logs/passC.start"
for c in "$D/chunks-c"/c-*.txt; do
  b=$(basename "$c" .txt)
  wget --input-file="$c" \
    --force-directories --directory-prefix="$D/raw" \
    --adjust-extension --no-verbose --no-clobber -e robots=off \
    --user-agent='feadulta-archiver/1.0 (+incident-183; mirror local)' \
    --wait=0.1 --tries=2 --timeout=45 --waitretry=3 \
    --reject-regex="$REJECT" \
    --output-file="$D/logs/wget-$b.log" &
done
wait
date -u +%FT%TZ > "$D/logs/passC.end"

echo "PASE C TERMINADO"
echo "ficheros en disco despues: $(find "$D/raw" -type f | wc -l)"
du -sh "$D/raw"
grep -hoE 'ERROR [0-9]+' "$D/logs"/wget-c-*.log | sort | uniq -c