#!/bin/bash
# Pase C: captura iterativa de las paginas internas enlazadas que no estaban en el inventario
# (rutas alternativas de menu, paginas de autor de K2, /anterior, ...).
# Itera hasta que no aparezcan URLs nuevas o hasta MAXIT vueltas.
set -uo pipefail
BASE=/home/rafa/joomla-migration/mirror-antiguo
RUN=$(cat "$BASE/CURRENT_RUN")
DIR=$BASE/runs/$RUN
MAXIT=${MAXIT:-6}
W=${WORKERS:-4}
REJECT='(\?|&)(start|limitstart|limit|print|tmpl|format|searchword|task|orderby|filter|catid|month|year)='

for it in $(seq 1 $MAXIT); do
  IN=$DIR/passC-$it-input.txt
  if [ "$it" = "1" ]; then
    cp "$DIR/missing-real.txt" "$IN"
  else
    # recalcular huecos con lo capturado hasta ahora
    python3 "$BASE/scripts/30-extract-links.py" > "$DIR/logs/extract-$it.log" 2>&1
    grep -v '?' "$DIR/missing-pages.txt" > "$DIR/missing-pages-sinquery.txt"
    python3 "$BASE/scripts/34-clasifica-missing.py" > "$DIR/logs/clasifica-$it.log" 2>&1
    cp "$DIR/missing-real.txt" "$IN"
  fi

  n=$(wc -l < "$IN")
  echo "=== iteracion $it: $n URLs candidatas ==="
  [ "$n" -eq 0 ] && { echo "no quedan huecos"; break; }

  mkdir -p "$DIR/chunks-c"
  rm -f "$DIR/chunks-c"/*.txt
  split -n l/$W -d --additional-suffix=.txt "$IN" "$DIR/chunks-c/c$it-"
  for c in "$DIR/chunks-c"/c$it-*.txt; do
    [ -s "$c" ] || continue
    b=$(basename "$c" .txt)
    wget --input-file="$c" \
      --force-directories --directory-prefix="$DIR/raw" \
      --adjust-extension --no-verbose --no-clobber -e robots=off \
      --user-agent='feadulta-archiver/1.0 (+incident-183; mirror local)' \
      --wait=0.1 --tries=2 --timeout=45 --waitretry=3 \
      --reject-regex="$REJECT" \
      --output-file="$DIR/logs/wget-$b.log" &
  done
  wait
  echo "  ficheros ahora: $(find "$DIR/raw" -type f | wc -l)"
  echo "  errores: $(grep -hoE 'ERROR [0-9]+' "$DIR/logs"/wget-c$it-*.log | sort | uniq -c | tr '\n' ' ')"
done

date -u +%FT%TZ > "$DIR/logs/passC.end"
echo "PASE C TERMINADO"
find "$DIR/raw" -type f | wc -l
du -sh "$DIR/raw"