${logLines.length === 0 ? '
Логов пока нет
' :
logLines.map(l => {
const escaped = l.replace(/&/g,'&').replace(/[$1]')
.replace(/\[([A-Z]{2}|local|\?\?)\]/, '
[$1]')
.replace(/((?:\d{1,3}\.){3}\d{1,3}|::ffff:[^\s]+)/, '
$1')
.replace(/\b(resized:)/, '
$1')
.replace(/\b(compress:)/, '
$1')
.replace(/\((\d+KB)\)/, '
($1)');
return '
' + colored + '
';
}).join('')}
`);
});
// ==================== URL Article Parser ====================
// Rate limit for parser: 60 req/min per IP
const parserLimiter = rateLimit({
windowMs: 60 * 1000,
max: 60,
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Слишком много запросов к парсеру. Попробуйте через минуту.' },
});
app.use(['/parse', '/metadata', '/text', '/preview'], parserLimiter);
// In-memory cache with TTL (6 hours)
const PARSER_CACHE_TTL = 6 * 60 * 60 * 1000;
const parserCache = new Map();
function getCached(key) {
const entry = parserCache.get(key);
if (!entry) return null;
if (Date.now() - entry.ts > PARSER_CACHE_TTL) {
parserCache.delete(key);
return null;
}
return entry.data;
}
function setCache(key, data) {
// Limit cache size to 500 entries
if (parserCache.size > 500) {
const oldest = parserCache.keys().next().value;
parserCache.delete(oldest);
}
parserCache.set(key, { data, ts: Date.now() });
}
// Clean expired cache entries every 30 min
setInterval(() => {
const now = Date.now();
for (const [key, entry] of parserCache) {
if (now - entry.ts > PARSER_CACHE_TTL) parserCache.delete(key);
}
}, 30 * 60 * 1000);
// Validate URL
function isValidParserUrl(urlStr) {
if (!urlStr || urlStr.length > 2048) return false;
try {
const u = new URL(urlStr);
if (!['http:', 'https:'].includes(u.protocol)) return false;
// Block localhost and private IPs
const host = u.hostname.toLowerCase();
if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '0.0.0.0') return false;
if (/^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/.test(host)) return false;
if (host.endsWith('.local') || host.endsWith('.internal')) return false;
return true;
} catch {
return false;
}
}
// Detect charset from content-type header or meta tags
function detectCharset(contentType, htmlBuffer) {
// From Content-Type header
if (contentType) {
const match = contentType.match(/charset=([^\s;]+)/i);
if (match) return match[1].trim().toLowerCase();
}
// From HTML meta tags (first 4KB)
const head = htmlBuffer.slice(0, 4096).toString('ascii');
const metaMatch = head.match(/charset=["']?([^"'\s;>]+)/i);
if (metaMatch) return metaMatch[1].trim().toLowerCase();
return 'utf-8';
}
// Fetch and parse article
async function fetchAndParse(url) {
const cached = getCached(url);
if (cached) return cached;
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 10000);
let response;
try {
response = await fetch(url, {
signal: controller.signal,
redirect: 'follow',
follow: 5,
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WAParser/1.0)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru,en;q=0.5',
},
});
} finally {
clearTimeout(timeout);
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const contentType = response.headers.get('content-type') || '';
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
throw new Error('URL не содержит HTML-страницу');
}
// Check size limit (2MB)
const contentLength = parseInt(response.headers.get('content-length') || '0', 10);
if (contentLength > 2 * 1024 * 1024) {
throw new Error('Страница слишком большая (>2MB)');
}
const arrayBuf = await response.arrayBuffer();
const htmlBuffer = Buffer.from(arrayBuf);
if (htmlBuffer.length > 2 * 1024 * 1024) {
throw new Error('Страница слишком большая (>2MB)');
}
// Decode to UTF-8
const charset = detectCharset(contentType, htmlBuffer);
let html;
if (charset === 'utf-8' || charset === 'utf8') {
html = htmlBuffer.toString('utf-8');
} else if (iconv.encodingExists(charset)) {
html = iconv.decode(htmlBuffer, charset);
} else {
html = htmlBuffer.toString('utf-8');
}
// Parse with JSDOM
const dom = new JSDOM(html, { url });
const doc = dom.window.document;
// Extract metadata
const getMeta = (name) => {
const el = doc.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
return el ? el.getAttribute('content') : null;
};
const metadata = {
og_title: getMeta('og:title'),
og_description: getMeta('og:description'),
og_image: getMeta('og:image'),
og_site_name: getMeta('og:site_name'),
description: getMeta('description'),
author: getMeta('author') || getMeta('article:author'),
date: getMeta('article:published_time') || getMeta('date') || getMeta('pubdate'),
};
// Parse with Readability
const reader = new Readability(doc);
const article = reader.parse();
// Extract images from parsed content
let images = [];
let links = [];
if (article && article.content) {
const contentDom = new JSDOM(article.content);
const contentDoc = contentDom.window.document;
contentDoc.querySelectorAll('img').forEach(img => {
const src = img.getAttribute('src') || img.getAttribute('data-src');
if (src) {
try {
images.push(new URL(src, url).href);
} catch {}
}
});
contentDoc.querySelectorAll('a[href]').forEach(a => {
const href = a.getAttribute('href');
if (href && href.startsWith('http')) {
links.push({ text: a.textContent.trim(), href });
}
});
contentDom.window.close();
}
// Clean content HTML: remove script, style, on* attrs, data-* attrs
let cleanHtml = article ? article.content : '';
if (cleanHtml) {
const cleanDom = new JSDOM(cleanHtml);
const cleanDoc = cleanDom.window.document;
cleanDoc.querySelectorAll('script, style, iframe, object, embed').forEach(el => el.remove());
cleanDoc.querySelectorAll('*').forEach(el => {
const attrs = Array.from(el.attributes);
attrs.forEach(attr => {
if (attr.name.startsWith('on') || attr.name.startsWith('data-')) {
el.removeAttribute(attr.name);
}
});
});
cleanHtml = cleanDoc.body.innerHTML;
cleanDom.window.close();
}
const contentText = article ? article.textContent : '';
const wordCount = contentText ? contentText.trim().split(/\s+/).filter(Boolean).length : 0;
const result = {
url,
title: article ? article.title : (metadata.og_title || doc.title || ''),
author: metadata.author || (article ? article.byline : null),
site: metadata.og_site_name || new URL(url).hostname,
date_published: metadata.date || null,
excerpt: article ? article.excerpt : (metadata.og_description || metadata.description || ''),
content_html: cleanHtml,
content_text: contentText,
lead_image: metadata.og_image || (images.length > 0 ? images[0] : null),
images: [...new Set(images)],
links,
word_count: wordCount,
lang: article ? article.lang : (doc.documentElement.lang || null),
};
dom.window.close();
setCache(url, result);
return result;
}
// Password generator
app.get('/password', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'password.html'));
});
// Editor
app.get('/editor', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'editor.html'));
});
// Dev utilities
app.get('/sanitizer', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'sanitizer.html'));
});
app.get('/converter', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'converter.html'));
});
app.get('/formatter', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'formatter.html'));
});
// Parser page
app.get('/parser', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'parser.html'));
});
// Home page alias
app.get('/home', (req, res) => res.sendFile(path.join(__dirname, 'public', 'index.html')));
// Image compressor page
app.get('/compress', (req, res) => res.sendFile(path.join(__dirname, 'public', 'compress.html')));
// Health check
app.get('/health', (req, res) => {
res.send('OK');
});
// Full parse
app.get('/parse', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL. Используйте http:// или https://' });
}
try {
const result = await fetchAndParse(url);
const country = getCountry(req.ip);
const msg = `[${country}] ${req.ip} parse: ${url} → "${result.title}" (${result.word_count} words)`;
console.log(msg);
logToFile(msg);
res.json(result);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка парсинга' });
}
});
// Metadata only
app.get('/metadata', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.json({
url: result.url,
title: result.title,
description: result.excerpt,
image: result.lead_image,
site: result.site,
author: result.author,
date_published: result.date_published,
lang: result.lang,
});
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Text only
app.get('/text', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.type('text/plain').send(result.content_text);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Preview card
app.get('/preview', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.json({
url: result.url,
title: result.title,
description: result.excerpt,
image: result.lead_image,
site: result.site,
});
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Multer error handler
app.use((err, req, res, next) => {
if (err instanceof multer.MulterError) {
if (err.code === 'LIMIT_FILE_SIZE') {
return res.status(413).json({ error: 'Файл слишком большой. Максимум 20MB.' });
}
if (err.code === 'LIMIT_FILE_COUNT') {
return res.status(400).json({ error: 'Слишком много файлов. Максимум 50.' });
}
return res.status(400).json({ error: err.message });
}
if (err) {
console.error('Unhandled error:', err);
return res.status(500).json({ error: 'Внутренняя ошибка сервера' });
}
next();
});
// HTTP Client history (per session)
app.get('/api/history', (req, res) => {
res.json(req.session.httpHistory || []);
});
app.post('/api/history/add', express.json(), (req, res) => {
if (!req.session.httpHistory) req.session.httpHistory = [];
req.session.httpHistory.unshift(req.body);
if (req.session.httpHistory.length > 30) req.session.httpHistory.length = 30;
res.json({ ok: true });
});
app.delete('/api/history', (req, res) => {
req.session.httpHistory = [];
res.json({ ok: true });
});
// HTTP Client proxy
const proxyLimiter = rateLimit({
windowMs: 60 * 1000,
max: 60,
message: { error: 'Слишком много запросов. Попробуйте через минуту.' },
});
app.post('/api/proxy', proxyLimiter, express.json({ limit: '10mb' }), async (req, res) => {
const { url, method = 'GET', headers = {}, body, timeout = 30000 } = req.body || {};
if (!url || typeof url !== 'string') {
return res.status(400).json({ error: 'URL обязателен' });
}
let parsedUrl;
try { parsedUrl = new URL(url); } catch {
return res.status(400).json({ error: 'Невалидный URL' });
}
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
return res.status(400).json({ error: 'Поддерживаются только http и https' });
}
const startTime = Date.now();
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.min(Number(timeout) || 30000, 60000));
const fetchOptions = {
method: method.toUpperCase(),
headers: Object.fromEntries(
Object.entries(headers).filter(([k]) => !['host','connection','transfer-encoding'].includes(k.toLowerCase()))
),
signal: controller.signal,
redirect: 'follow',
};
if (body != null && !['GET', 'HEAD'].includes(method.toUpperCase())) {
fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
}
const response = await fetch(url, fetchOptions);
clearTimeout(timer);
const elapsed = Date.now() - startTime;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const MAX_SIZE = 10 * 1024 * 1024;
const reader = response.body?.getReader();
const chunks = [];
let totalSize = 0;
let truncated = false;
if (reader) {
while (true) {
const { done, value } = await reader.read();
if (done) break;
totalSize += value.length;
if (totalSize > MAX_SIZE) { truncated = true; break; }
chunks.push(value);
}
}
const buffer = Buffer.concat(chunks.map(c => Buffer.from(c)));
const responseBody = buffer.toString('utf-8');
res.json({
status: response.status,
statusText: response.statusText,
headers: responseHeaders,
body: responseBody,
time: elapsed,
size: totalSize,
truncated,
url: response.url,
});
} catch (err) {
const elapsed = Date.now() - startTime;
if (err.name === 'AbortError') {
return res.status(408).json({ error: 'Таймаут запроса', time: elapsed });
}
res.status(500).json({ error: err.message, time: elapsed });
}
});
app.get('/httpclient', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'httpclient.html'));
});
app.get('/redirects', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'redirects.html'));
});
// ── Redirect Chain Analyzer ──────────────────────────────────────────────────
const redirectLimiter = rateLimit({ windowMs: 60_000, max: 20 });
const UA_STRINGS = {
desktop: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
mobile: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
};
app.post('/api/redirect-analyze', redirectLimiter, express.json(), async (req, res) => {
const { url: rawUrl, userAgent = 'desktop', method = 'GET' } = req.body || {};
if (!rawUrl || typeof rawUrl !== 'string') {
return res.status(400).json({ error: 'URL обязателен' });
}
let parsedUrl;
try {
parsedUrl = new URL(rawUrl);
} catch {
return res.status(400).json({ error: 'Некорректный URL. Используйте http:// или https://' });
}
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
return res.status(400).json({ error: 'Поддерживаются только протоколы http и https' });
}
const uaString = UA_STRINGS[userAgent] || UA_STRINGS.desktop;
const MAX_STEPS = 15;
const STEP_TIMEOUT = 10_000; // ms
const chain = [];
const visitedUrls = new Set();
let currentUrl = rawUrl;
let loopDetected = false;
const totalStart = Date.now();
for (let step = 0; step < MAX_STEPS; step++) {
if (visitedUrls.has(currentUrl)) {
loopDetected = true;
break;
}
visitedUrls.add(currentUrl);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), STEP_TIMEOUT);
const stepStart = Date.now();
try {
const response = await fetch(currentUrl, {
method: method.toUpperCase(),
redirect: 'manual',
signal: controller.signal,
headers: {
'User-Agent': uaString,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
},
});
clearTimeout(timer);
const stepTime = Date.now() - stepStart;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const location = response.headers.get('location') || null;
chain.push({
url: currentUrl,
status: response.status,
statusText: response.statusText,
location,
time: stepTime,
headers: responseHeaders,
});
// Not a redirect — this is the final destination
if (response.status < 300 || response.status >= 400) {
break;
}
if (!location) break;
// Resolve relative redirects
try {
currentUrl = new URL(location, currentUrl).href;
} catch {
break;
}
} catch (err) {
clearTimeout(timer);
const stepTime = Date.now() - stepStart;
chain.push({
url: currentUrl,
status: 0,
statusText: err.name === 'AbortError' ? 'Timeout' : err.message,
location: null,
time: stepTime,
headers: {},
});
break;
}
}
const totalTime = Date.now() - totalStart;
const finalUrl = chain.length > 0 ? chain[chain.length - 1].url : rawUrl;
// Detect issues
const issues = [];
const redirectSteps = chain.filter(s => s.status >= 300 && s.status < 400);
if (redirectSteps.length > 3) {
issues.push('long_chain');
}
// Check https→http downgrade
for (let i = 0; i < chain.length - 1; i++) {
const curr = chain[i];
const next = chain[i + 1];
if (curr.url.startsWith('https://') && next.url.startsWith('http://')) {
issues.push('mixed_protocol');
break;
}
}
// Check 302 used instead of 301
if (chain.some(s => s.status === 302)) {
issues.push('302_not_301');
}
res.json({
chain,
final_url: finalUrl,
total_time: totalTime,
loop_detected: loopDetected,
issues,
});
});
// ── SVG Editor ──────────────────────────────────────────────────────────────────
app.get('/svgeditor', (req, res) => {
res.sendFile(require('path').join(__dirname, 'public', 'svgeditor.html'));
});
app.post('/api/svg-optimize', express.json({ limit: '5mb' }), (req, res) => {
let svg = req.body.svg || '';
const originalLen = svg.length;
// Remove XML declaration
svg = svg.replace(/<\?xml[^?]*\?>/g, '');
// Remove comments
svg = svg.replace(//g, '');
// Remove metadata, title, desc
svg = svg.replace(/