rafa
66ee943da4
El enumerador de URLs no se había commiteado y se perdió. Recuperado reproduciendo la composición de la corrida previa (menús×5 idiomas + categorías con contenido + contenido no-ES). Se añade también classify_gaps.py (clasifica huecos: traducir vs descargar) y run_all.sh. Outputs y .venv quedan en .gitignore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
72 lines
3.1 KiB
JavaScript
72 lines
3.1 KiB
JavaScript
// Capa 0+1 (crawl): status, URL final (cross-lang), html lang, texto visible, enlaces salientes.
|
|
// Resumible: añade a report.jsonl; al rearrancar salta lo ya hecho. Coste 0 (local).
|
|
const { chromium } = require('playwright');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const DIR = __dirname;
|
|
const URLS = JSON.parse(fs.readFileSync(path.join(DIR,'urls.json'),'utf8'));
|
|
const OUT = path.join(DIR,'report.jsonl');
|
|
const CONCURRENCY = parseInt(process.env.CONC||'3',10);
|
|
const LIMIT = parseInt(process.env.LIMIT||'0',10); // 0 = todas
|
|
|
|
// resumir
|
|
const done = new Set();
|
|
if (fs.existsSync(OUT)) for (const l of fs.readFileSync(OUT,'utf8').split('\n')) {
|
|
if(!l.trim()) continue; try { done.add(JSON.parse(l).key); } catch(e){}
|
|
}
|
|
let tasks = URLS.map(u=>({...u, key:u.lang+'|'+u.url})).filter(t=>!done.has(t.key));
|
|
if (LIMIT) tasks = tasks.slice(0,LIMIT);
|
|
console.log(`pendientes: ${tasks.length} (ya hechas ${done.size})`);
|
|
|
|
const langPrefix = (url) => {
|
|
const m = url.match(/\/fea\/(es|en|fr|it|pt)\//); if (m) return m[1];
|
|
// sin prefijo bajo /fea/ = es (hide_default)
|
|
if (/\/fea\//.test(url)) return 'es';
|
|
return '';
|
|
};
|
|
|
|
(async () => {
|
|
const browser = await chromium.launch({ args:['--ignore-certificate-errors'] });
|
|
const ctx = await browser.newContext({ ignoreHTTPSErrors:true, userAgent:'fea-verify-crawler' });
|
|
const out = fs.createWriteStream(OUT, { flags:'a' });
|
|
let i=0, done2=0;
|
|
const worker = async () => {
|
|
const page = await ctx.newPage();
|
|
while (i < tasks.length) {
|
|
const t = tasks[i++];
|
|
const rec = { key:t.key, url:t.url, lang:t.lang, kind:t.kind, id:t.id };
|
|
let resp=null, lastErr=null;
|
|
for (let a=0; a<3 && !resp; a++) {
|
|
try { resp = await page.goto(t.url, { waitUntil:'domcontentloaded', timeout:30000 }); }
|
|
catch(e){ lastErr=e; await page.waitForTimeout(700); }
|
|
}
|
|
try {
|
|
if (!resp) throw lastErr || new Error('no response');
|
|
rec.status = resp ? resp.status() : 0;
|
|
rec.final = page.url();
|
|
rec.redirected = rec.final !== t.url;
|
|
rec.final_lang = langPrefix(rec.final);
|
|
const data = await page.evaluate(() => {
|
|
const main = document.querySelector('main, .wp-block-post-content, .entry-content, article') || document.body;
|
|
const text = (main.innerText||'').replace(/\s+/g,' ').trim().slice(0,1200);
|
|
const htmlLang = document.documentElement.getAttribute('lang')||'';
|
|
const links = Array.from(document.querySelectorAll('a[href]')).map(a=>a.href)
|
|
.filter(h=>/^https?:/.test(h));
|
|
return { text, htmlLang, links:[...new Set(links)] };
|
|
});
|
|
rec.text = data.text; rec.htmlLang = data.htmlLang; rec.links = data.links;
|
|
} catch (e) {
|
|
rec.status = -1; rec.error = String(e).slice(0,120); rec.final=t.url; rec.links=[]; rec.text='';
|
|
}
|
|
out.write(JSON.stringify(rec)+'\n');
|
|
if (++done2 % 100 === 0) console.log(` ${done2}/${tasks.length}`);
|
|
}
|
|
await page.close();
|
|
};
|
|
await Promise.all(Array.from({length:CONCURRENCY}, worker));
|
|
out.end();
|
|
await browser.close();
|
|
console.log(`crawl terminado: ${done2} nuevas, total report = ${done.size+done2}`);
|
|
})();
|