#!/bin/bash
# Pase A: captura de las paginas HTML del inventario. Sin recursion, sin page-requisites.
set -uo pipefail
BASE=/home/rafa/joomla-migration/mirror-antiguo
RUN=$(cat "$BASE/CURRENT_RUN")
DIR=$BASE/runs/$RUN
cd "$DIR"

UA='feadulta-archiver/1.0 (+incident-183; mirror local)'
REJECT='(\?|&)(start|limitstart|limit|print|tmpl|format|searchword|task|orderby|filter|catid|month|year)='

date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/logs/passA.start"

for c in chunks/urls-*.txt; do
  n=$(basename "$c" .txt)
  wget \
    --input-file="$c" \
    --force-directories --directory-prefix="$DIR/raw" \
    --adjust-extension \
    --no-verbose \
    -e robots=off \
    --user-agent="$UA" \
    --wait=0.15 --tries=3 --timeout=45 --waitretry=5 \
    --reject-regex="$REJECT" \
    --no-check-certificate \
    --output-file="$DIR/logs/wget-$n.log" &
  echo "$!" >> "$DIR/logs/passA.pids"
done

wait
date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/logs/passA.end"
echo "PASE A TERMINADO"
find "$DIR/raw" -type f | wc -l
du -sh "$DIR/raw"