wa-dev-tools/routes/parser.js

const express = require('express');
const { JSDOM } = require('jsdom');
const { Readability } = require('@mozilla/readability');
const iconv = require('iconv-lite');
const geoip = require('geoip-lite');
const log = require('../lib/logger');
const { validateUrl } = require('../lib/ssrf');

const router = express.Router();

// In-memory cache (max 50 entries, 10 min TTL)
const cache = new Map();
const CACHE_TTL = 10 * 60 * 1000;
const CACHE_MAX = 50;

function getCached(url) {
  const entry = cache.get(url);
  if (!entry) return null;
  if (Date.now() - entry.ts > CACHE_TTL) { cache.delete(url); return null; }
  return entry.data;
}

function setCache(url, data) {
  if (cache.size >= CACHE_MAX) {
    const oldest = cache.keys().next().value;
    cache.delete(oldest);
  }
  cache.set(url, { data, ts: Date.now() });
}

function getCountry(ip) {
  const clean = ip.replace(/^::ffff:/, '');
  if (clean === '127.0.0.1' || clean === '::1') return 'local';
  const geo = geoip.lookup(clean);
  return geo ? geo.country : '??';
}

function detectCharset(contentType, htmlBuffer) {
  if (contentType) {
    const match = contentType.match(/charset=([^\s;]+)/i);
    if (match) return match[1].trim().toLowerCase();
  }
  const head = htmlBuffer.slice(0, 4096).toString('ascii');
  const metaMatch = head.match(/charset=["']?([^"'\s;>]+)/i);
  if (metaMatch) return metaMatch[1].trim().toLowerCase();
  return 'utf-8';
}

async function fetchAndParse(url) {
  const cached = getCached(url);
  if (cached) return cached;

  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), 10000);

  let response;
  try {
    response = await fetch(url, {
      signal: controller.signal,
      redirect: 'follow',
      follow: 5,
      headers: {
        'User-Agent': 'Mozilla/5.0 (compatible; WAParser/1.0)',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'ru,en;q=0.5',
      },
    });
  } finally {
    clearTimeout(timeout);
  }

  if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);

  const contentType = response.headers.get('content-type') || '';
  if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
    throw new Error('URL не содержит HTML-страницу');
  }

  const contentLength = parseInt(response.headers.get('content-length') || '0', 10);
  if (contentLength > 2 * 1024 * 1024) throw new Error('Страница слишком большая (>2MB)');

  const arrayBuf = await response.arrayBuffer();
  const htmlBuffer = Buffer.from(arrayBuf);
  if (htmlBuffer.length > 2 * 1024 * 1024) throw new Error('Страница слишком большая (>2MB)');

  const charset = detectCharset(contentType, htmlBuffer);
  let html;
  if (charset === 'utf-8' || charset === 'utf8') html = htmlBuffer.toString('utf-8');
  else if (iconv.encodingExists(charset)) html = iconv.decode(htmlBuffer, charset);
  else html = htmlBuffer.toString('utf-8');

  const dom = new JSDOM(html, { url });
  const doc = dom.window.document;

  const getMeta = (name) => {
    const el = doc.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
    return el ? el.getAttribute('content') : null;
  };

  const metadata = {
    og_title: getMeta('og:title'),
    og_description: getMeta('og:description'),
    og_image: getMeta('og:image'),
    og_site_name: getMeta('og:site_name'),
    description: getMeta('description'),
    author: getMeta('author') || getMeta('article:author'),
    date: getMeta('article:published_time') || getMeta('date') || getMeta('pubdate'),
  };

  const reader = new Readability(doc);
  const article = reader.parse();

  let images = [];
  let links = [];
  if (article && article.content) {
    const contentDom = new JSDOM(article.content);
    const contentDoc = contentDom.window.document;
    contentDoc.querySelectorAll('img').forEach(img => {
      const src = img.getAttribute('src') || img.getAttribute('data-src');
      if (src) { try { images.push(new URL(src, url).href); } catch {} }
    });
    contentDoc.querySelectorAll('a[href]').forEach(a => {
      const href = a.getAttribute('href');
      if (href && href.startsWith('http')) links.push({ text: a.textContent.trim(), href });
    });
    contentDom.window.close();
  }

  let cleanHtml = article ? article.content : '';
  if (cleanHtml) {
    const cleanDom = new JSDOM(cleanHtml);
    const cleanDoc = cleanDom.window.document;
    cleanDoc.querySelectorAll('script, style, iframe, object, embed').forEach(el => el.remove());
    cleanDoc.querySelectorAll('*').forEach(el => {
      Array.from(el.attributes).forEach(attr => {
        if (attr.name.startsWith('on') || attr.name.startsWith('data-')) el.removeAttribute(attr.name);
      });
    });
    cleanHtml = cleanDoc.body.innerHTML;
    cleanDom.window.close();
  }

  const contentText = article ? article.textContent : '';
  const wordCount = contentText ? contentText.trim().split(/\s+/).filter(Boolean).length : 0;

  const result = {
    url,
    title: article ? article.title : (metadata.og_title || doc.title || ''),
    author: metadata.author || (article ? article.byline : null),
    site: metadata.og_site_name || new URL(url).hostname,
    date_published: metadata.date || null,
    excerpt: article ? article.excerpt : (metadata.og_description || metadata.description || ''),
    content_html: cleanHtml,
    content_text: contentText,
    lead_image: metadata.og_image || (images.length > 0 ? images[0] : null),
    images: [...new Set(images)],
    links,
    word_count: wordCount,
    lang: article ? article.lang : (doc.documentElement.lang || null),
  };

  dom.window.close();
  setCache(url, result);
  return result;
}

// Full parse
router.get('/parse', async (req, res) => {
  const check = await validateUrl(req.query.url);
  if (!check.safe) return res.status(400).json({ error: check.error });
  try {
    const result = await fetchAndParse(check.url);
    const country = getCountry(req.ip);
    const msg = `[${country}] ${req.ip} parse: ${check.url} → "${result.title}" (${result.word_count} words)`;
    log.info(msg);
    log.logToFile(msg);
    res.json(result);
  } catch (err) {
    res.status(500).json({ error: err.message || 'Ошибка парсинга' });
  }
});

// Metadata only
router.get('/metadata', async (req, res) => {
  const check = await validateUrl(req.query.url);
  if (!check.safe) return res.status(400).json({ error: check.error });
  try {
    const result = await fetchAndParse(check.url);
    res.json({
      url: result.url, title: result.title, description: result.excerpt,
      image: result.lead_image, site: result.site, author: result.author,
      date_published: result.date_published, lang: result.lang,
    });
  } catch (err) {
    res.status(500).json({ error: err.message || 'Ошибка' });
  }
});

// Text only
router.get('/text', async (req, res) => {
  const check = await validateUrl(req.query.url);
  if (!check.safe) return res.status(400).json({ error: check.error });
  try {
    const result = await fetchAndParse(check.url);
    res.type('text/plain').send(result.content_text);
  } catch (err) {
    res.status(500).json({ error: err.message || 'Ошибка' });
  }
});

// Preview card
router.get('/preview', async (req, res) => {
  const check = await validateUrl(req.query.url);
  if (!check.safe) return res.status(400).json({ error: check.error });
  try {
    const result = await fetchAndParse(check.url);
    res.json({ url: result.url, title: result.title, description: result.excerpt, image: result.lead_image, site: result.site });
  } catch (err) {
    res.status(500).json({ error: err.message || 'Ошибка' });
  }
});

// Parser page
router.get('/parser', (req, res) => {
  res.sendFile(require('path').join(__dirname, '..', 'public', 'parser.html'));
});

module.exports = router;