// Capa 0+1 (crawl): status, URL final (cross-lang), html lang, texto visible, enlaces salientes.
// Resumible: añade a report.jsonl; al rearrancar salta lo ya hecho. Coste 0 (local).
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');

const DIR = __dirname;
const URLS = JSON.parse(fs.readFileSync(path.join(DIR,'urls.json'),'utf8'));
const OUT = path.join(DIR,'report.jsonl');
const CONCURRENCY = parseInt(process.env.CONC||'3',10);
const LIMIT = parseInt(process.env.LIMIT||'0',10); // 0 = todas

// resumir
const done = new Set();
if (fs.existsSync(OUT)) for (const l of fs.readFileSync(OUT,'utf8').split('\n')) {
  if(!l.trim()) continue; try { done.add(JSON.parse(l).key); } catch(e){}
}
let tasks = URLS.map(u=>({...u, key:u.lang+'|'+u.url})).filter(t=>!done.has(t.key));
if (LIMIT) tasks = tasks.slice(0,LIMIT);
console.log(`pendientes: ${tasks.length} (ya hechas ${done.size})`);

const langPrefix = (url) => {
  const m = url.match(/\/fea\/(es|en|fr|it|pt)\//); if (m) return m[1];
  // sin prefijo bajo /fea/ = es (hide_default)
  if (/\/fea\//.test(url)) return 'es';
  return '';
};

(async () => {
  const browser = await chromium.launch({ args:['--ignore-certificate-errors'] });
  const ctx = await browser.newContext({ ignoreHTTPSErrors:true, userAgent:'fea-verify-crawler' });
  const out = fs.createWriteStream(OUT, { flags:'a' });
  let i=0, done2=0;
  const worker = async () => {
    const page = await ctx.newPage();
    while (i < tasks.length) {
      const t = tasks[i++];
      const rec = { key:t.key, url:t.url, lang:t.lang, kind:t.kind, id:t.id };
      let resp=null, lastErr=null;
      for (let a=0; a<3 && !resp; a++) {
        try { resp = await page.goto(t.url, { waitUntil:'domcontentloaded', timeout:30000 }); }
        catch(e){ lastErr=e; await page.waitForTimeout(700); }
      }
      try {
        if (!resp) throw lastErr || new Error('no response');
        rec.status = resp ? resp.status() : 0;
        rec.final = page.url();
        rec.redirected = rec.final !== t.url;
        rec.final_lang = langPrefix(rec.final);
        const data = await page.evaluate(() => {
          const main = document.querySelector('main, .wp-block-post-content, .entry-content, article') || document.body;
          const text = (main.innerText||'').replace(/\s+/g,' ').trim().slice(0,1200);
          const htmlLang = document.documentElement.getAttribute('lang')||'';
          const links = Array.from(document.querySelectorAll('a[href]')).map(a=>a.href)
            .filter(h=>/^https?:/.test(h));
          return { text, htmlLang, links:[...new Set(links)] };
        });
        rec.text = data.text; rec.htmlLang = data.htmlLang; rec.links = data.links;
      } catch (e) {
        rec.status = -1; rec.error = String(e).slice(0,120); rec.final=t.url; rec.links=[]; rec.text='';
      }
      out.write(JSON.stringify(rec)+'\n');
      if (++done2 % 100 === 0) console.log(`  ${done2}/${tasks.length}`);
    }
    await page.close();
  };
  await Promise.all(Array.from({length:CONCURRENCY}, worker));
  out.end();
  await browser.close();
  console.log(`crawl terminado: ${done2} nuevas, total report = ${done.size+done2}`);
})();