chore(verify): recuperar enum_urls.php y commitear tooling de verificación #121

El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-22 08:30:18 -04:00
parent 863a3c6677
commit 66ee943da4
10 changed files with 582 additions and 0 deletions
@@ -0,0 +1,71 @@
+// Capa 0+1 (crawl): status, URL final (cross-lang), html lang, texto visible, enlaces salientes.
+// Resumible: añade a report.jsonl; al rearrancar salta lo ya hecho. Coste 0 (local).
+const { chromium } = require('playwright');
+const fs = require('fs');
+const path = require('path');
+
+const DIR = __dirname;
+const URLS = JSON.parse(fs.readFileSync(path.join(DIR,'urls.json'),'utf8'));
+const OUT = path.join(DIR,'report.jsonl');
+const CONCURRENCY = parseInt(process.env.CONC||'3',10);
+const LIMIT = parseInt(process.env.LIMIT||'0',10); // 0 = todas
+
+// resumir
+const done = new Set();
+if (fs.existsSync(OUT)) for (const l of fs.readFileSync(OUT,'utf8').split('\n')) {
+  if(!l.trim()) continue; try { done.add(JSON.parse(l).key); } catch(e){}
+}
+let tasks = URLS.map(u=>({...u, key:u.lang+'|'+u.url})).filter(t=>!done.has(t.key));
+if (LIMIT) tasks = tasks.slice(0,LIMIT);
+console.log(`pendientes: ${tasks.length} (ya hechas ${done.size})`);
+
+const langPrefix = (url) => {
+  const m = url.match(/\/fea\/(es|en|fr|it|pt)\//); if (m) return m[1];
+  // sin prefijo bajo /fea/ = es (hide_default)
+  if (/\/fea\//.test(url)) return 'es';
+  return '';
+};
+
+(async () => {
+  const browser = await chromium.launch({ args:['--ignore-certificate-errors'] });
+  const ctx = await browser.newContext({ ignoreHTTPSErrors:true, userAgent:'fea-verify-crawler' });
+  const out = fs.createWriteStream(OUT, { flags:'a' });
+  let i=0, done2=0;
+  const worker = async () => {
+    const page = await ctx.newPage();
+    while (i < tasks.length) {
+      const t = tasks[i++];
+      const rec = { key:t.key, url:t.url, lang:t.lang, kind:t.kind, id:t.id };
+      let resp=null, lastErr=null;
+      for (let a=0; a<3 && !resp; a++) {
+        try { resp = await page.goto(t.url, { waitUntil:'domcontentloaded', timeout:30000 }); }
+        catch(e){ lastErr=e; await page.waitForTimeout(700); }
+      }
+      try {
+        if (!resp) throw lastErr || new Error('no response');
+        rec.status = resp ? resp.status() : 0;
+        rec.final = page.url();
+        rec.redirected = rec.final !== t.url;
+        rec.final_lang = langPrefix(rec.final);
+        const data = await page.evaluate(() => {
+          const main = document.querySelector('main, .wp-block-post-content, .entry-content, article') || document.body;
+          const text = (main.innerText||'').replace(/\s+/g,' ').trim().slice(0,1200);
+          const htmlLang = document.documentElement.getAttribute('lang')||'';
+          const links = Array.from(document.querySelectorAll('a[href]')).map(a=>a.href)
+            .filter(h=>/^https?:/.test(h));
+          return { text, htmlLang, links:[...new Set(links)] };
+        });
+        rec.text = data.text; rec.htmlLang = data.htmlLang; rec.links = data.links;
+      } catch (e) {
+        rec.status = -1; rec.error = String(e).slice(0,120); rec.final=t.url; rec.links=[]; rec.text='';
+      }
+      out.write(JSON.stringify(rec)+'\n');
+      if (++done2 % 100 === 0) console.log(`  ${done2}/${tasks.length}`);
+    }
+    await page.close();
+  };
+  await Promise.all(Array.from({length:CONCURRENCY}, worker));
+  out.end();
+  await browser.close();
+  console.log(`crawl terminado: ${done2} nuevas, total report = ${done.size+done2}`);
+})();