#!/bin/bash
# Lote de prueba de 200 URLs para validar la invocacion de wget y el arbol resultante
set -uo pipefail
BASE=/home/rafa/joomla-migration/mirror-antiguo
SM=$BASE/smoke
rm -rf "$SM"; mkdir -p "$SM/raw"
shuf -n 200 "$BASE/inventory/urls-input.txt" > "$SM/urls.txt"
# aseguramos la portada y una carta
echo 'http://antiguo.feadulta.com/es/' >> "$SM/urls.txt"

t0=$(date +%s)
wget --input-file="$SM/urls.txt" \
  --force-directories --directory-prefix="$SM/raw" \
  --adjust-extension --no-verbose -e robots=off \
  --user-agent='feadulta-archiver/1.0 (+incident-183; mirror local)' \
  --wait=0.15 --tries=3 --timeout=45 --waitretry=5 \
  --reject-regex='(\?|&)(start|limitstart|limit|print|tmpl|format|searchword|task|orderby|filter|catid|month|year)=' \
  --output-file="$SM/wget.log"
t1=$(date +%s)

echo "=== tiempo: $((t1-t0))s para $(wc -l < "$SM/urls.txt") URLs ==="
echo "=== ficheros ==="; find "$SM/raw" -type f | wc -l
echo "=== arbol (muestra) ==="; find "$SM/raw" -type f | head -8
echo "=== errores en el log ==="; grep -icE 'error|failed' "$SM/wget.log" || true
grep -iE 'error|failed' "$SM/wget.log" | head -10
echo "=== tamano ==="; du -sh "$SM/raw"
echo "=== 500 en apache durante el lote ==="
docker logs --since "${t0}" joomla-mirror-web 2>&1 | grep -c '" 500 ' || true