265 lines
9.6 KiB
JavaScript
265 lines
9.6 KiB
JavaScript
const express = require('express');
|
|
const { JSDOM } = require('jsdom');
|
|
const { Readability } = require('@mozilla/readability');
|
|
const iconv = require('iconv-lite');
|
|
const geoip = require('geoip-lite');
|
|
const log = require('../lib/logger');
|
|
const { validateUrl } = require('../lib/ssrf');
|
|
|
|
const rateLimit = require('express-rate-limit');
|
|
|
|
const router = express.Router();
|
|
|
|
// Rate limit parser endpoints (prevent DDoS via server)
|
|
const parserLimiter = rateLimit({
|
|
keyGenerator: (req) => (req.session && req.session.user && req.session.user.id) ? "user_" + req.session.user.id : req.ip, windowMs: 60000, max: 20, message: { error: 'Слишком много запросов' } });
|
|
|
|
// In-memory cache (max 50 entries, 10 min TTL)
|
|
const cache = new Map();
|
|
const CACHE_TTL = 10 * 60 * 1000;
|
|
const CACHE_MAX = 50;
|
|
|
|
function getCached(url) {
|
|
const entry = cache.get(url);
|
|
if (!entry) return null;
|
|
if (Date.now() - entry.ts > CACHE_TTL) { cache.delete(url); return null; }
|
|
return entry.data;
|
|
}
|
|
|
|
function setCache(url, data) {
|
|
if (cache.size >= CACHE_MAX) {
|
|
const oldest = cache.keys().next().value;
|
|
cache.delete(oldest);
|
|
}
|
|
cache.set(url, { data, ts: Date.now() });
|
|
}
|
|
|
|
function getCountry(ip) {
|
|
const clean = ip.replace(/^::ffff:/, '');
|
|
if (clean === '127.0.0.1' || clean === '::1') return 'local';
|
|
const geo = geoip.lookup(clean);
|
|
return geo ? geo.country : '??';
|
|
}
|
|
|
|
function detectCharset(contentType, htmlBuffer) {
|
|
if (contentType) {
|
|
const match = contentType.match(/charset=([^\s;]+)/i);
|
|
if (match) return match[1].trim().toLowerCase();
|
|
}
|
|
const head = htmlBuffer.slice(0, 4096).toString('ascii');
|
|
const metaMatch = head.match(/charset=["']?([^"'\s;>]+)/i);
|
|
if (metaMatch) return metaMatch[1].trim().toLowerCase();
|
|
return 'utf-8';
|
|
}
|
|
|
|
async function fetchAndParse(url) {
|
|
const cached = getCached(url);
|
|
if (cached) return cached;
|
|
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), 10000);
|
|
|
|
let response;
|
|
try {
|
|
// Follow redirects manually, re-validating each hop against SSRF rules.
|
|
const MAX_REDIRECTS = 5;
|
|
let currentUrl = url;
|
|
for (let i = 0; ; i++) {
|
|
response = await fetch(currentUrl, {
|
|
signal: controller.signal,
|
|
redirect: 'manual',
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (compatible; WAParser/1.0)',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'ru,en;q=0.5',
|
|
},
|
|
});
|
|
if (![301, 302, 303, 307, 308].includes(response.status)) break;
|
|
if (i >= MAX_REDIRECTS) throw new Error('Слишком много редиректов');
|
|
const loc = response.headers.get('location');
|
|
if (!loc) break;
|
|
let nextUrl;
|
|
try { nextUrl = new URL(loc, currentUrl).href; } catch { throw new Error('Некорректный редирект'); }
|
|
const rcheck = await validateUrl(nextUrl);
|
|
if (!rcheck.safe) throw new Error('Редирект на запрещённый адрес');
|
|
currentUrl = rcheck.url;
|
|
}
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
|
|
if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
|
|
const contentType = response.headers.get('content-type') || '';
|
|
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
|
|
throw new Error('URL не содержит HTML-страницу');
|
|
}
|
|
|
|
const contentLength = parseInt(response.headers.get('content-length') || '0', 10);
|
|
if (contentLength > 2 * 1024 * 1024) throw new Error('Страница слишком большая (>2MB)');
|
|
|
|
// Stream the body and abort as soon as the 2MB cap is exceeded (avoids buffering huge/unbounded responses)
|
|
const MAX_HTML = 2 * 1024 * 1024;
|
|
const bodyReader = response.body?.getReader();
|
|
const chunks = [];
|
|
let total = 0;
|
|
if (bodyReader) {
|
|
while (true) {
|
|
const { done, value } = await bodyReader.read();
|
|
if (done) break;
|
|
total += value.length;
|
|
if (total > MAX_HTML) { try { await bodyReader.cancel(); } catch {} throw new Error('Страница слишком большая (>2MB)'); }
|
|
chunks.push(value);
|
|
}
|
|
}
|
|
const htmlBuffer = Buffer.concat(chunks.map((c) => Buffer.from(c)));
|
|
|
|
const charset = detectCharset(contentType, htmlBuffer);
|
|
let html;
|
|
if (charset === 'utf-8' || charset === 'utf8') html = htmlBuffer.toString('utf-8');
|
|
else if (iconv.encodingExists(charset)) html = iconv.decode(htmlBuffer, charset);
|
|
else html = htmlBuffer.toString('utf-8');
|
|
|
|
const dom = new JSDOM(html, { url });
|
|
const doc = dom.window.document;
|
|
|
|
const getMeta = (name) => {
|
|
const el = doc.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
|
|
return el ? el.getAttribute('content') : null;
|
|
};
|
|
|
|
const metadata = {
|
|
og_title: getMeta('og:title'),
|
|
og_description: getMeta('og:description'),
|
|
og_image: getMeta('og:image'),
|
|
og_site_name: getMeta('og:site_name'),
|
|
description: getMeta('description'),
|
|
author: getMeta('author') || getMeta('article:author'),
|
|
date: getMeta('article:published_time') || getMeta('date') || getMeta('pubdate'),
|
|
};
|
|
|
|
const reader = new Readability(doc);
|
|
const article = reader.parse();
|
|
|
|
let images = [];
|
|
let links = [];
|
|
if (article && article.content) {
|
|
const contentDom = new JSDOM(article.content);
|
|
const contentDoc = contentDom.window.document;
|
|
contentDoc.querySelectorAll('img').forEach(img => {
|
|
const src = img.getAttribute('src') || img.getAttribute('data-src');
|
|
if (src) { try { images.push(new URL(src, url).href); } catch {} }
|
|
});
|
|
contentDoc.querySelectorAll('a[href]').forEach(a => {
|
|
const href = a.getAttribute('href');
|
|
if (href && href.startsWith('http')) links.push({ text: a.textContent.trim(), href });
|
|
});
|
|
contentDom.window.close();
|
|
}
|
|
|
|
let cleanHtml = article ? article.content : '';
|
|
if (cleanHtml) {
|
|
const cleanDom = new JSDOM(cleanHtml);
|
|
const cleanDoc = cleanDom.window.document;
|
|
cleanDoc.querySelectorAll('script, style, iframe, object, embed').forEach(el => el.remove());
|
|
cleanDoc.querySelectorAll('*').forEach(el => {
|
|
Array.from(el.attributes).forEach(attr => {
|
|
const n = attr.name.toLowerCase();
|
|
if (n.startsWith('on') || n.startsWith('data-')) { el.removeAttribute(attr.name); return; }
|
|
// Strip dangerous URL schemes from href/src/action/xlink:href (javascript:, data:, vbscript:)
|
|
if (['href', 'src', 'action', 'formaction', 'xlink:href'].includes(n) && /^\s*(javascript|data|vbscript):/i.test(attr.value)) {
|
|
el.removeAttribute(attr.name);
|
|
}
|
|
});
|
|
});
|
|
cleanHtml = cleanDoc.body.innerHTML;
|
|
cleanDom.window.close();
|
|
}
|
|
|
|
const contentText = article ? article.textContent : '';
|
|
const wordCount = contentText ? contentText.trim().split(/\s+/).filter(Boolean).length : 0;
|
|
|
|
const result = {
|
|
url,
|
|
title: article ? article.title : (metadata.og_title || doc.title || ''),
|
|
author: metadata.author || (article ? article.byline : null),
|
|
site: metadata.og_site_name || new URL(url).hostname,
|
|
date_published: metadata.date || null,
|
|
excerpt: article ? article.excerpt : (metadata.og_description || metadata.description || ''),
|
|
content_html: cleanHtml,
|
|
content_text: contentText,
|
|
lead_image: metadata.og_image || (images.length > 0 ? images[0] : null),
|
|
images: [...new Set(images)],
|
|
links,
|
|
word_count: wordCount,
|
|
lang: article ? article.lang : (doc.documentElement.lang || null),
|
|
};
|
|
|
|
dom.window.close();
|
|
setCache(url, result);
|
|
return result;
|
|
}
|
|
|
|
// Full parse
|
|
router.get('/parse', parserLimiter, async (req, res) => {
|
|
const check = await validateUrl(req.query.url);
|
|
if (!check.safe) return res.status(400).json({ error: check.error });
|
|
try {
|
|
const result = await fetchAndParse(check.url);
|
|
const country = getCountry(req.ip);
|
|
const msg = `[${country}] ${req.ip} parse: ${check.url} → "${result.title}" (${result.word_count} words)`;
|
|
log.info(msg);
|
|
log.logToFile(msg);
|
|
res.json(result);
|
|
} catch (err) {
|
|
res.status(500).json({ error: err.message || 'Ошибка парсинга' });
|
|
}
|
|
});
|
|
|
|
// Metadata only
|
|
router.get('/metadata', parserLimiter, async (req, res) => {
|
|
const check = await validateUrl(req.query.url);
|
|
if (!check.safe) return res.status(400).json({ error: check.error });
|
|
try {
|
|
const result = await fetchAndParse(check.url);
|
|
res.json({
|
|
url: result.url, title: result.title, description: result.excerpt,
|
|
image: result.lead_image, site: result.site, author: result.author,
|
|
date_published: result.date_published, lang: result.lang,
|
|
});
|
|
} catch (err) {
|
|
res.status(500).json({ error: err.message || 'Ошибка' });
|
|
}
|
|
});
|
|
|
|
// Text only
|
|
router.get('/text', parserLimiter, async (req, res) => {
|
|
const check = await validateUrl(req.query.url);
|
|
if (!check.safe) return res.status(400).json({ error: check.error });
|
|
try {
|
|
const result = await fetchAndParse(check.url);
|
|
res.type('text/plain').send(result.content_text);
|
|
} catch (err) {
|
|
res.status(500).json({ error: err.message || 'Ошибка' });
|
|
}
|
|
});
|
|
|
|
// Preview card
|
|
router.get('/preview', parserLimiter, async (req, res) => {
|
|
const check = await validateUrl(req.query.url);
|
|
if (!check.safe) return res.status(400).json({ error: check.error });
|
|
try {
|
|
const result = await fetchAndParse(check.url);
|
|
res.json({ url: result.url, title: result.title, description: result.excerpt, image: result.lead_image, site: result.site });
|
|
} catch (err) {
|
|
res.status(500).json({ error: err.message || 'Ошибка' });
|
|
}
|
|
});
|
|
|
|
// Parser page
|
|
router.get('/parser', (req, res) => {
|
|
res.sendFile(require('path').join(__dirname, '..', 'public', 'parser.html'));
|
|
});
|
|
|
|
module.exports = router;
|