wa-dev-tools/server.js
2026-03-21 22:06:15 +03:00

1112 lines
44 KiB
JavaScript
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

const express = require('express');
const session = require('express-session');
const multer = require('multer');
const sharp = require('sharp');
const fs = require('fs');
const path = require('path');
const archiver = require('archiver');
const rateLimit = require('express-rate-limit');
const geoip = require('geoip-lite');
const { JSDOM } = require('jsdom');
const { Readability } = require('@mozilla/readability');
const iconv = require('iconv-lite');
const app = express();
app.set('trust proxy', 1);
app.use(session({
secret: 'wa-dev-tools-secret-2024',
resave: false,
saveUninitialized: true,
cookie: { maxAge: 30 * 24 * 60 * 60 * 1000 }, // 30 days
}));
app.use(express.static('public'));
// Rate limiting — 30 requests per minute per IP
const limiter = rateLimit({
windowMs: 60 * 1000,
max: 30,
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Слишком много запросов. Попробуйте через минуту.' },
});
app.use('/compress', limiter);
const upload = multer({
dest: 'uploads/',
limits: { fileSize: 20 * 1024 * 1024 }, // 20MB per file
fileFilter: (req, file, cb) => {
const validTypes = ['image/jpeg', 'image/png', 'image/webp'];
if (validTypes.includes(file.mimetype)) cb(null, true);
else cb(new Error('Unsupported file type'));
}
});
// Periodic cleanup of uploads/ dir — files older than 10 minutes
const UPLOADS_DIR = path.join(__dirname, 'uploads');
const DOWNLOADS_DIR = path.join(__dirname, 'downloads');
for (const dir of [UPLOADS_DIR, DOWNLOADS_DIR]) {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}
function cleanupDir(dir, maxAgeMs) {
try {
const files = fs.readdirSync(dir);
const now = Date.now();
for (const file of files) {
const filePath = path.join(dir, file);
try {
const stat = fs.statSync(filePath);
if (now - stat.mtimeMs > maxAgeMs) {
fs.unlinkSync(filePath);
}
} catch {}
}
} catch {}
}
// Clean uploads every 2 min (10 min age), downloads every 5 min (30 min age)
setInterval(() => cleanupDir(UPLOADS_DIR, 10 * 60 * 1000), 2 * 60 * 1000);
setInterval(() => cleanupDir(DOWNLOADS_DIR, 30 * 60 * 1000), 5 * 60 * 1000);
function getCountry(ip) {
const clean = ip.replace(/^::ffff:/, '');
if (clean === '127.0.0.1' || clean === '::1') return 'local';
const geo = geoip.lookup(clean);
return geo ? geo.country : '??';
}
function logToFile(text) {
const timestamp = new Date().toISOString();
const logLine = `[${timestamp}] ${text}\n`;
fs.appendFileSync(path.join(__dirname, 'compress.log'), logLine);
}
// Cyrillic to Latin transliteration
const TRANSLIT_MAP = {
'а':'a','б':'b','в':'v','г':'g','д':'d','е':'e','ё':'yo','ж':'zh','з':'z','и':'i',
'й':'y','к':'k','л':'l','м':'m','н':'n','о':'o','п':'p','р':'r','с':'s','т':'t',
'у':'u','ф':'f','х':'kh','ц':'ts','ч':'ch','ш':'sh','щ':'shch','ъ':'','ы':'y',
'ь':'','э':'e','ю':'yu','я':'ya',
'А':'A','Б':'B','В':'V','Г':'G','Д':'D','Е':'E','Ё':'Yo','Ж':'Zh','З':'Z','И':'I',
'Й':'Y','К':'K','Л':'L','М':'M','Н':'N','О':'O','П':'P','Р':'R','С':'S','Т':'T',
'У':'U','Ф':'F','Х':'Kh','Ц':'Ts','Ч':'Ch','Ш':'Sh','Щ':'Shch','Ъ':'','Ы':'Y',
'Ь':'','Э':'E','Ю':'Yu','Я':'Ya',
};
function fixMulterFilename(str) {
// Multer/busboy may give us UTF-8 bytes decoded as Latin-1
try {
const buf = Buffer.from(str, 'latin1');
const decoded = buf.toString('utf8');
// If it decoded cleanly and contains actual Cyrillic, use it
if (/[а-яёА-ЯЁ]/.test(decoded)) return decoded;
} catch {}
return str;
}
function transliterate(str) {
const fixed = fixMulterFilename(str);
return fixed.replace(/[а-яёА-ЯЁ]/g, ch => TRANSLIT_MAP[ch] !== undefined ? TRANSLIT_MAP[ch] : ch)
.replace(/[^\w.\-]/g, '_')
.replace(/_+/g, '_')
.replace(/^_|_$/g, '');
}
// Format extension map
function getOutputExtension(format, originalName) {
if (format === 'webp') return '.webp';
if (format === 'jpeg') return '.jpg';
if (format === 'png') return '.png';
// original — keep current extension
return path.extname(originalName);
}
function changeExtension(filename, newExt) {
const base = path.basename(filename, path.extname(filename));
return base + newExt;
}
app.post('/compress', upload.array('images', 50), async (req, res) => {
const resize = parseInt(req.body.resize || '0', 10);
const format = (req.body.format || 'original').toLowerCase();
const validFormats = ['original', 'webp', 'jpeg', 'png'];
if (!validFormats.includes(format)) {
return res.status(400).json({ error: 'Неверный формат. Допустимые: original, webp, jpeg, png' });
}
if (!req.files || !req.files.length) {
return res.status(400).json({ error: 'Файлы не загружены' });
}
const archiveName = `archive_${Date.now()}.zip`;
const archivePath = path.join(DOWNLOADS_DIR, archiveName);
const output = fs.createWriteStream(archivePath);
const archive = archiver('zip', { zlib: { level: 9 } });
const stats = [];
const country = getCountry(req.ip);
try {
archive.pipe(output);
for (const file of req.files) {
const inputPath = file.path;
const originalSize = file.size;
const image = sharp(inputPath);
const metadata = await image.metadata();
// Resize if needed
if (
resize > 0 &&
metadata.width &&
metadata.height &&
Math.max(metadata.width, metadata.height) > resize
) {
const scale = resize / Math.max(metadata.width, metadata.height);
const newWidth = Math.round(metadata.width * scale);
const newHeight = Math.round(metadata.height * scale);
image.resize({ width: newWidth, height: newHeight });
const msg = `[${country}] ${req.ip} resized: ${file.originalname} ${metadata.width}x${metadata.height}${newWidth}x${newHeight} (${(originalSize / 1024).toFixed(0)}KB)`;
console.log(msg);
logToFile(msg);
} else {
const msg = `[${country}] ${req.ip} compress: ${file.originalname} ${metadata.width}x${metadata.height} format=${format} (${(originalSize / 1024).toFixed(0)}KB)`;
console.log(msg);
logToFile(msg);
}
// Convert/compress based on format
let buffer;
if (format === 'webp') {
buffer = await image.webp({ quality: 60 }).toBuffer();
} else if (format === 'jpeg') {
buffer = await image.jpeg({ quality: 60 }).toBuffer();
} else if (format === 'png') {
buffer = await image.png({ compressionLevel: 9 }).toBuffer();
} else {
// original — keep source format
if (file.mimetype === 'image/jpeg') {
buffer = await image.jpeg({ quality: 60 }).toBuffer();
} else if (file.mimetype === 'image/png') {
buffer = await image.png({ compressionLevel: 9 }).toBuffer();
} else if (file.mimetype === 'image/webp') {
buffer = await image.webp({ quality: 60 }).toBuffer();
} else {
continue;
}
}
const readableName = fixMulterFilename(file.originalname);
const newExt = getOutputExtension(format, readableName);
const rawName = format === 'original'
? readableName
: changeExtension(readableName, newExt);
const outputName = transliterate(rawName);
archive.append(buffer, { name: outputName });
const compressedSize = buffer.length;
const savings = originalSize > 0
? Math.round((1 - compressedSize / originalSize) * 100)
: 0;
stats.push({
filename: readableName,
outputFilename: outputName,
originalSize,
compressedSize,
savings,
});
}
await archive.finalize();
await new Promise((resolve, reject) => {
output.on('close', resolve);
output.on('error', reject);
});
res.json({
success: true,
downloadUrl: `/download/${archiveName}`,
stats,
});
} catch (err) {
console.error('Compression error:', err);
// Clean up partial archive
try { fs.existsSync(archivePath) && fs.unlinkSync(archivePath); } catch {}
res.status(500).json({ error: 'Ошибка при сжатии' });
} finally {
if (req.files) {
req.files.forEach(file => {
try { fs.existsSync(file.path) && fs.unlinkSync(file.path); } catch {}
});
}
}
});
// Download endpoint
app.get('/download/:filename', (req, res) => {
const filename = path.basename(req.params.filename); // prevent path traversal
const filePath = path.join(DOWNLOADS_DIR, filename);
if (!fs.existsSync(filePath)) {
return res.status(404).json({ error: 'Файл не найден' });
}
res.download(filePath, 'compressed.zip');
});
// Markdown viewer
app.get('/md', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'md.html'));
});
// Placeholder page
app.get('/placeholder', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'placeholder.html'));
});
// Placeholder image generator (via sharp + SVG)
function normalizeColor(color) {
if (!color) return '#cccccc';
if (/^[0-9A-Fa-f]{6}$/.test(color)) return '#' + color;
if (/^[0-9A-Fa-f]{3}$/.test(color)) return '#' + color;
return color.toLowerCase();
}
function escapeXml(str) {
return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
}
async function placeholderHandler(req, res) {
const { size, bg, fg, text: textFromPath } = req.params;
const { text: textFromQuery, format = 'png', fontsize } = req.query;
const text = textFromQuery || textFromPath || '';
const [w, h] = size.split('x').map(Number);
const width = Math.min(Math.max(w || 300, 1), 4000);
const height = Math.min(Math.max(h || 200, 1), 4000);
const bgColor = normalizeColor(bg);
const fgColor = normalizeColor(fg);
const fontSize = fontsize ? parseInt(fontsize, 10) : Math.max(12, Math.floor(Math.min(width, height) / 8));
const svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">
<rect width="100%" height="100%" fill="${escapeXml(bgColor)}"/>
<text x="50%" y="50%" dominant-baseline="central" text-anchor="middle"
font-family="sans-serif" font-weight="bold" font-size="${fontSize}"
fill="${escapeXml(fgColor)}">${escapeXml(text)}</text>
</svg>`;
try {
let img = sharp(Buffer.from(svg));
if (format === 'jpg' || format === 'jpeg') {
res.type('jpeg').end(await img.jpeg({ quality: 90 }).toBuffer());
} else if (format === 'webp') {
res.type('webp').end(await img.webp({ quality: 90 }).toBuffer());
} else {
res.type('png').end(await img.png().toBuffer());
}
} catch (err) {
console.error('Placeholder error:', err);
res.status(500).json({ error: 'Generation failed' });
}
}
app.get('/placeholder-img/:size/:bg/:fg', placeholderHandler);
app.get('/placeholder-img/:size/:bg/:fg/:text', placeholderHandler);
// Logs page
const LOG_FILE = path.join(__dirname, 'compress.log');
app.get('/logs', (req, res) => {
const lines = parseInt(req.query.lines || '100', 10);
const filter = (req.query.filter || '').toLowerCase();
let logLines = [];
try {
const content = fs.readFileSync(LOG_FILE, 'utf8');
logLines = content.trim().split('\n').filter(Boolean);
} catch { /* no log file yet */ }
if (filter) {
logLines = logLines.filter(l => l.toLowerCase().includes(filter));
}
// Last N lines, newest first
logLines = logLines.slice(-Math.min(lines, 1000)).reverse();
res.send(`<!DOCTYPE html>
<html lang="ru" class="dark">
<head>
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Логи — images.wadevelop.ru</title>
<link href="/vendor/fonts.css" rel="stylesheet">
<style>
*{margin:0;padding:0;box-sizing:border-box}
body{font-family:'JetBrains Mono',monospace;font-size:13px;background:#060c18;color:#c0c0c0;padding:16px}
a{color:#0054e6;text-decoration:none}
.header{display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:16px}
.header h1{font-size:16px;color:#fff;font-weight:600}
.header input,.header select{background:#091020;border:1px solid #162040;color:#c0c0c0;padding:6px 10px;border-radius:6px;font-family:inherit;font-size:12px}
.header input:focus,.header select:focus{outline:none;border-color:#0054e6}
.badge{background:#162040;color:#888;padding:3px 8px;border-radius:4px;font-size:11px}
.log-line{padding:6px 10px;border-bottom:1px solid #091020;line-height:1.5;white-space:pre-wrap;word-break:break-all}
.log-line:hover{background:#091020}
.ts{color:#555}
.country{color:#0054e6;font-weight:600}
.ip{color:#888}
.action-resize{color:#ffd600}
.action-compress{color:#6b9fff}
.size{color:#888}
.empty{text-align:center;padding:40px;color:#555}
.auto{margin-left:auto;display:flex;align-items:center;gap:6px;font-size:11px;color:#555}
.auto input{width:14px;height:14px}
/* Sidebar */
.sidebar{position:fixed;top:0;left:0;width:56px;height:100vh;display:flex;flex-direction:column;align-items:center;padding:16px 0 12px;z-index:50;border-right:1px solid #162040;background:rgba(9,16,32,0.8);backdrop-filter:blur(12px)}
.sidebar-logo{margin-bottom:24px;color:#666;font-size:10px;font-weight:700;letter-spacing:.08em}
.sidebar-nav{display:flex;flex-direction:column;gap:6px;flex:1}
.sidebar-link{width:40px;height:40px;border-radius:10px;display:flex;align-items:center;justify-content:center;color:#666;transition:all .2s;text-decoration:none;position:relative}
.sidebar-link:hover{color:#c0c0c0;background:rgba(255,255,255,.04)}
.sidebar-link.active{color:#0054e6;background:rgba(0,84,230,.08)}
.sidebar-link.active::before{content:'';position:absolute;left:-8px;top:50%;transform:translateY(-50%);width:3px;height:20px;border-radius:0 3px 3px 0;background:#0054e6}
.main-content{margin-left:56px}
@media(max-width:640px){
.sidebar{position:fixed;top:auto;bottom:0;left:0;width:100%;height:52px;flex-direction:row;justify-content:center;padding:0 16px;border-right:none;border-top:1px solid #162040;gap:0}
.sidebar-logo{display:none}
.sidebar-nav{flex-direction:row;gap:4px;flex:none}
.sidebar-link.active::before{left:50%;top:auto;bottom:-6px;transform:translateX(-50%);width:20px;height:3px;border-radius:3px 3px 0 0}
.main-content{margin-left:0;padding-bottom:60px}
}
</style>
</head>
<body>
<nav class="sidebar">
<div class="sidebar-logo">WA</div>
<div class="sidebar-nav">
<a href="/" class="sidebar-link" title="Главная"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M2.25 12l8.954-8.955c.44-.439 1.152-.439 1.591 0L21.75 12M4.5 9.75v10.125c0 .621.504 1.125 1.125 1.125H9.75v-4.875c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125V21h4.125c.621 0 1.125-.504 1.125-1.125V9.75M8.25 21h8.25"/></svg></a>
<a href="/compress" class="sidebar-link" title="Картинки"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M2.25 15.75l5.159-5.159a2.25 2.25 0 013.182 0l5.159 5.159m-1.5-1.5l1.409-1.409a2.25 2.25 0 013.182 0l2.909 2.909M3.75 21h16.5A2.25 2.25 0 0023.25 18.75V5.25A2.25 2.25 0 0021 3H3.75A2.25 2.25 0 001.5 5.25v13.5A2.25 2.25 0 003.75 21z"/></svg></a>
<a href="/md" class="sidebar-link" title="Markdown"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M19.5 14.25v-2.625a3.375 3.375 0 00-3.375-3.375h-1.5A1.125 1.125 0 0113.5 7.125v-1.5a3.375 3.375 0 00-3.375-3.375H8.25m0 12.75h7.5m-7.5 3H12M10.5 2.25H5.625c-.621 0-1.125.504-1.125 1.125v17.25c0 .621.504 1.125 1.125 1.125h12.75c.621 0 1.125-.504 1.125-1.125V11.25a9 9 0 00-9-9z"/></svg></a>
<a href="/logs" class="sidebar-link active" title="Логи"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 12h16.5m-16.5 3.75h16.5M3.75 19.5h16.5M5.625 4.5h12.75a1.875 1.875 0 010 3.75H5.625a1.875 1.875 0 010-3.75z"/></svg></a>
<a href="/parser" class="sidebar-link" title="Parser"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M13.19 8.688a4.5 4.5 0 011.242 7.244l-4.5 4.5a4.5 4.5 0 01-6.364-6.364l1.757-1.757m9.86-2.06a4.5 4.5 0 00-6.364-6.364L6.257 6.514a4.5 4.5 0 001.242 7.244"/></svg></a>
<a href="/password" class="sidebar-link" title="Password"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M15.75 5.25a3 3 0 013 3m3 0a6 6 0 01-7.029 5.912c-.563-.097-1.159.026-1.563.43L10.5 17.25H8.25v2.25H6v2.25H2.25v-2.818c0-.597.237-1.17.659-1.591l6.499-6.499c.404-.404.527-1 .43-1.563A6 6 0 1121.75 8.25z"/></svg></a>
<a href="/sanitizer" class="sidebar-link" title="Sanitizer"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M9 12.75L11.25 15 15 9.75m-3-7.036A11.959 11.959 0 013.598 6 11.99 11.99 0 003 9.749c0 5.592 3.824 10.29 9 11.623 5.176-1.332 9-6.03 9-11.622 0-1.31-.21-2.571-.598-3.751h-.152c-3.196 0-6.1-1.248-8.25-3.285z"/></svg></a>
<a href="/converter" class="sidebar-link" title="Converter"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M7.5 21L3 16.5m0 0L7.5 12M3 16.5h13.5m0-13.5L21 7.5m0 0L16.5 12M21 7.5H7.5"/></svg></a>
<a href="/formatter" class="sidebar-link" title="Formatter"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M17.25 6.75L22.5 12l-5.25 5.25m-10.5 0L1.5 12l5.25-5.25m7.5-3l-4.5 16.5"/></svg></a>
<a href="/editor" class="sidebar-link" title="Editor"><svg width="22" height="22" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L10.582 16.07a4.5 4.5 0 01-1.897 1.13L6 18l.8-2.685a4.5 4.5 0 011.13-1.897l8.932-8.931zm0 0L19.5 7.125M18 14v4.75A2.25 2.25 0 0115.75 21H5.25A2.25 2.25 0 013 18.75V8.25A2.25 2.25 0 015.25 6H10"/></svg></a>
</div>
</nav>
<div class="main-content">
<div class="header">
<h1>Логи</h1>
<span class="badge">${logLines.length} записей</span>
<input type="text" id="filter" placeholder="Фильтр..." value="${filter.replace(/"/g,'&quot;')}">
<select id="lines">
<option ${lines===50?'selected':''} value="50">50</option>
<option ${lines===100?'selected':''} value="100">100</option>
<option ${lines===300?'selected':''} value="300">300</option>
<option ${lines===1000?'selected':''} value="1000">1000</option>
</select>
<div class="auto"><input type="checkbox" id="autoRefresh"><label for="autoRefresh">Авто (5с)</label></div>
</div>
<div id="logs">
${logLines.length === 0 ? '<div class="empty">Логов пока нет</div>' :
logLines.map(l => {
const escaped = l.replace(/&/g,'&amp;').replace(/</g,'&lt;');
const colored = escaped
.replace(/^\[([^\]]+)\]/, '<span class="ts">[$1]</span>')
.replace(/\[([A-Z]{2}|local|\?\?)\]/, '<span class="country">[$1]</span>')
.replace(/((?:\d{1,3}\.){3}\d{1,3}|::ffff:[^\s]+)/, '<span class="ip">$1</span>')
.replace(/\b(resized:)/, '<span class="action-resize">$1</span>')
.replace(/\b(compress:)/, '<span class="action-compress">$1</span>')
.replace(/\((\d+KB)\)/, '<span class="size">($1)</span>');
return '<div class="log-line">' + colored + '</div>';
}).join('')}
</div>
<script>
const f=document.getElementById('filter'),s=document.getElementById('lines'),a=document.getElementById('autoRefresh');
function go(){location.search='?lines='+s.value+(f.value?'&filter='+encodeURIComponent(f.value):'')}
s.onchange=go;
f.addEventListener('keydown',e=>{if(e.key==='Enter')go()});
let t;a.onchange=()=>{clearInterval(t);if(a.checked)t=setInterval(()=>location.reload(),5000)};
</script>
</div>
</body>
</html>`);
});
// ==================== URL Article Parser ====================
// Rate limit for parser: 60 req/min per IP
const parserLimiter = rateLimit({
windowMs: 60 * 1000,
max: 60,
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Слишком много запросов к парсеру. Попробуйте через минуту.' },
});
app.use(['/parse', '/metadata', '/text', '/preview'], parserLimiter);
// In-memory cache with TTL (6 hours)
const PARSER_CACHE_TTL = 6 * 60 * 60 * 1000;
const parserCache = new Map();
function getCached(key) {
const entry = parserCache.get(key);
if (!entry) return null;
if (Date.now() - entry.ts > PARSER_CACHE_TTL) {
parserCache.delete(key);
return null;
}
return entry.data;
}
function setCache(key, data) {
// Limit cache size to 500 entries
if (parserCache.size > 500) {
const oldest = parserCache.keys().next().value;
parserCache.delete(oldest);
}
parserCache.set(key, { data, ts: Date.now() });
}
// Clean expired cache entries every 30 min
setInterval(() => {
const now = Date.now();
for (const [key, entry] of parserCache) {
if (now - entry.ts > PARSER_CACHE_TTL) parserCache.delete(key);
}
}, 30 * 60 * 1000);
// Validate URL
function isValidParserUrl(urlStr) {
if (!urlStr || urlStr.length > 2048) return false;
try {
const u = new URL(urlStr);
if (!['http:', 'https:'].includes(u.protocol)) return false;
// Block localhost and private IPs
const host = u.hostname.toLowerCase();
if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '0.0.0.0') return false;
if (/^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/.test(host)) return false;
if (host.endsWith('.local') || host.endsWith('.internal')) return false;
return true;
} catch {
return false;
}
}
// Detect charset from content-type header or meta tags
function detectCharset(contentType, htmlBuffer) {
// From Content-Type header
if (contentType) {
const match = contentType.match(/charset=([^\s;]+)/i);
if (match) return match[1].trim().toLowerCase();
}
// From HTML meta tags (first 4KB)
const head = htmlBuffer.slice(0, 4096).toString('ascii');
const metaMatch = head.match(/charset=["']?([^"'\s;>]+)/i);
if (metaMatch) return metaMatch[1].trim().toLowerCase();
return 'utf-8';
}
// Fetch and parse article
async function fetchAndParse(url) {
const cached = getCached(url);
if (cached) return cached;
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 10000);
let response;
try {
response = await fetch(url, {
signal: controller.signal,
redirect: 'follow',
follow: 5,
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WAParser/1.0)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru,en;q=0.5',
},
});
} finally {
clearTimeout(timeout);
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const contentType = response.headers.get('content-type') || '';
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
throw new Error('URL не содержит HTML-страницу');
}
// Check size limit (2MB)
const contentLength = parseInt(response.headers.get('content-length') || '0', 10);
if (contentLength > 2 * 1024 * 1024) {
throw new Error('Страница слишком большая (>2MB)');
}
const arrayBuf = await response.arrayBuffer();
const htmlBuffer = Buffer.from(arrayBuf);
if (htmlBuffer.length > 2 * 1024 * 1024) {
throw new Error('Страница слишком большая (>2MB)');
}
// Decode to UTF-8
const charset = detectCharset(contentType, htmlBuffer);
let html;
if (charset === 'utf-8' || charset === 'utf8') {
html = htmlBuffer.toString('utf-8');
} else if (iconv.encodingExists(charset)) {
html = iconv.decode(htmlBuffer, charset);
} else {
html = htmlBuffer.toString('utf-8');
}
// Parse with JSDOM
const dom = new JSDOM(html, { url });
const doc = dom.window.document;
// Extract metadata
const getMeta = (name) => {
const el = doc.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
return el ? el.getAttribute('content') : null;
};
const metadata = {
og_title: getMeta('og:title'),
og_description: getMeta('og:description'),
og_image: getMeta('og:image'),
og_site_name: getMeta('og:site_name'),
description: getMeta('description'),
author: getMeta('author') || getMeta('article:author'),
date: getMeta('article:published_time') || getMeta('date') || getMeta('pubdate'),
};
// Parse with Readability
const reader = new Readability(doc);
const article = reader.parse();
// Extract images from parsed content
let images = [];
let links = [];
if (article && article.content) {
const contentDom = new JSDOM(article.content);
const contentDoc = contentDom.window.document;
contentDoc.querySelectorAll('img').forEach(img => {
const src = img.getAttribute('src') || img.getAttribute('data-src');
if (src) {
try {
images.push(new URL(src, url).href);
} catch {}
}
});
contentDoc.querySelectorAll('a[href]').forEach(a => {
const href = a.getAttribute('href');
if (href && href.startsWith('http')) {
links.push({ text: a.textContent.trim(), href });
}
});
contentDom.window.close();
}
// Clean content HTML: remove script, style, on* attrs, data-* attrs
let cleanHtml = article ? article.content : '';
if (cleanHtml) {
const cleanDom = new JSDOM(cleanHtml);
const cleanDoc = cleanDom.window.document;
cleanDoc.querySelectorAll('script, style, iframe, object, embed').forEach(el => el.remove());
cleanDoc.querySelectorAll('*').forEach(el => {
const attrs = Array.from(el.attributes);
attrs.forEach(attr => {
if (attr.name.startsWith('on') || attr.name.startsWith('data-')) {
el.removeAttribute(attr.name);
}
});
});
cleanHtml = cleanDoc.body.innerHTML;
cleanDom.window.close();
}
const contentText = article ? article.textContent : '';
const wordCount = contentText ? contentText.trim().split(/\s+/).filter(Boolean).length : 0;
const result = {
url,
title: article ? article.title : (metadata.og_title || doc.title || ''),
author: metadata.author || (article ? article.byline : null),
site: metadata.og_site_name || new URL(url).hostname,
date_published: metadata.date || null,
excerpt: article ? article.excerpt : (metadata.og_description || metadata.description || ''),
content_html: cleanHtml,
content_text: contentText,
lead_image: metadata.og_image || (images.length > 0 ? images[0] : null),
images: [...new Set(images)],
links,
word_count: wordCount,
lang: article ? article.lang : (doc.documentElement.lang || null),
};
dom.window.close();
setCache(url, result);
return result;
}
// Password generator
app.get('/password', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'password.html'));
});
// Editor
app.get('/editor', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'editor.html'));
});
// Dev utilities
app.get('/sanitizer', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'sanitizer.html'));
});
app.get('/converter', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'converter.html'));
});
app.get('/formatter', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'formatter.html'));
});
// Parser page
app.get('/parser', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'parser.html'));
});
// Home page alias
app.get('/home', (req, res) => res.sendFile(path.join(__dirname, 'public', 'index.html')));
// Image compressor page
app.get('/compress', (req, res) => res.sendFile(path.join(__dirname, 'public', 'compress.html')));
// Health check
app.get('/health', (req, res) => {
res.send('OK');
});
// Full parse
app.get('/parse', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL. Используйте http:// или https://' });
}
try {
const result = await fetchAndParse(url);
const country = getCountry(req.ip);
const msg = `[${country}] ${req.ip} parse: ${url} → "${result.title}" (${result.word_count} words)`;
console.log(msg);
logToFile(msg);
res.json(result);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка парсинга' });
}
});
// Metadata only
app.get('/metadata', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.json({
url: result.url,
title: result.title,
description: result.excerpt,
image: result.lead_image,
site: result.site,
author: result.author,
date_published: result.date_published,
lang: result.lang,
});
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Text only
app.get('/text', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.type('text/plain').send(result.content_text);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Preview card
app.get('/preview', async (req, res) => {
const url = req.query.url;
if (!isValidParserUrl(url)) {
return res.status(400).json({ error: 'Невалидный URL' });
}
try {
const result = await fetchAndParse(url);
res.json({
url: result.url,
title: result.title,
description: result.excerpt,
image: result.lead_image,
site: result.site,
});
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Multer error handler
app.use((err, req, res, next) => {
if (err instanceof multer.MulterError) {
if (err.code === 'LIMIT_FILE_SIZE') {
return res.status(413).json({ error: 'Файл слишком большой. Максимум 20MB.' });
}
if (err.code === 'LIMIT_FILE_COUNT') {
return res.status(400).json({ error: 'Слишком много файлов. Максимум 50.' });
}
return res.status(400).json({ error: err.message });
}
if (err) {
console.error('Unhandled error:', err);
return res.status(500).json({ error: 'Внутренняя ошибка сервера' });
}
next();
});
// HTTP Client history (per session)
app.get('/api/history', (req, res) => {
res.json(req.session.httpHistory || []);
});
app.post('/api/history/add', express.json(), (req, res) => {
if (!req.session.httpHistory) req.session.httpHistory = [];
req.session.httpHistory.unshift(req.body);
if (req.session.httpHistory.length > 30) req.session.httpHistory.length = 30;
res.json({ ok: true });
});
app.delete('/api/history', (req, res) => {
req.session.httpHistory = [];
res.json({ ok: true });
});
// HTTP Client proxy
const proxyLimiter = rateLimit({
windowMs: 60 * 1000,
max: 60,
message: { error: 'Слишком много запросов. Попробуйте через минуту.' },
});
app.post('/api/proxy', proxyLimiter, express.json({ limit: '10mb' }), async (req, res) => {
const { url, method = 'GET', headers = {}, body, timeout = 30000 } = req.body || {};
if (!url || typeof url !== 'string') {
return res.status(400).json({ error: 'URL обязателен' });
}
let parsedUrl;
try { parsedUrl = new URL(url); } catch {
return res.status(400).json({ error: 'Невалидный URL' });
}
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
return res.status(400).json({ error: 'Поддерживаются только http и https' });
}
const startTime = Date.now();
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.min(Number(timeout) || 30000, 60000));
const fetchOptions = {
method: method.toUpperCase(),
headers: Object.fromEntries(
Object.entries(headers).filter(([k]) => !['host','connection','transfer-encoding'].includes(k.toLowerCase()))
),
signal: controller.signal,
redirect: 'follow',
};
if (body != null && !['GET', 'HEAD'].includes(method.toUpperCase())) {
fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
}
const response = await fetch(url, fetchOptions);
clearTimeout(timer);
const elapsed = Date.now() - startTime;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const MAX_SIZE = 10 * 1024 * 1024;
const reader = response.body?.getReader();
const chunks = [];
let totalSize = 0;
let truncated = false;
if (reader) {
while (true) {
const { done, value } = await reader.read();
if (done) break;
totalSize += value.length;
if (totalSize > MAX_SIZE) { truncated = true; break; }
chunks.push(value);
}
}
const buffer = Buffer.concat(chunks.map(c => Buffer.from(c)));
const responseBody = buffer.toString('utf-8');
res.json({
status: response.status,
statusText: response.statusText,
headers: responseHeaders,
body: responseBody,
time: elapsed,
size: totalSize,
truncated,
url: response.url,
});
} catch (err) {
const elapsed = Date.now() - startTime;
if (err.name === 'AbortError') {
return res.status(408).json({ error: 'Таймаут запроса', time: elapsed });
}
res.status(500).json({ error: err.message, time: elapsed });
}
});
app.get('/httpclient', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'httpclient.html'));
});
app.get('/redirects', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'redirects.html'));
});
// ── Redirect Chain Analyzer ──────────────────────────────────────────────────
const redirectLimiter = rateLimit({ windowMs: 60_000, max: 20 });
const UA_STRINGS = {
desktop: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
mobile: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
};
app.post('/api/redirect-analyze', redirectLimiter, express.json(), async (req, res) => {
const { url: rawUrl, userAgent = 'desktop', method = 'GET' } = req.body || {};
if (!rawUrl || typeof rawUrl !== 'string') {
return res.status(400).json({ error: 'URL обязателен' });
}
let parsedUrl;
try {
parsedUrl = new URL(rawUrl);
} catch {
return res.status(400).json({ error: 'Некорректный URL. Используйте http:// или https://' });
}
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
return res.status(400).json({ error: 'Поддерживаются только протоколы http и https' });
}
const uaString = UA_STRINGS[userAgent] || UA_STRINGS.desktop;
const MAX_STEPS = 15;
const STEP_TIMEOUT = 10_000; // ms
const chain = [];
const visitedUrls = new Set();
let currentUrl = rawUrl;
let loopDetected = false;
const totalStart = Date.now();
for (let step = 0; step < MAX_STEPS; step++) {
if (visitedUrls.has(currentUrl)) {
loopDetected = true;
break;
}
visitedUrls.add(currentUrl);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), STEP_TIMEOUT);
const stepStart = Date.now();
try {
const response = await fetch(currentUrl, {
method: method.toUpperCase(),
redirect: 'manual',
signal: controller.signal,
headers: {
'User-Agent': uaString,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
},
});
clearTimeout(timer);
const stepTime = Date.now() - stepStart;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const location = response.headers.get('location') || null;
chain.push({
url: currentUrl,
status: response.status,
statusText: response.statusText,
location,
time: stepTime,
headers: responseHeaders,
});
// Not a redirect — this is the final destination
if (response.status < 300 || response.status >= 400) {
break;
}
if (!location) break;
// Resolve relative redirects
try {
currentUrl = new URL(location, currentUrl).href;
} catch {
break;
}
} catch (err) {
clearTimeout(timer);
const stepTime = Date.now() - stepStart;
chain.push({
url: currentUrl,
status: 0,
statusText: err.name === 'AbortError' ? 'Timeout' : err.message,
location: null,
time: stepTime,
headers: {},
});
break;
}
}
const totalTime = Date.now() - totalStart;
const finalUrl = chain.length > 0 ? chain[chain.length - 1].url : rawUrl;
// Detect issues
const issues = [];
const redirectSteps = chain.filter(s => s.status >= 300 && s.status < 400);
if (redirectSteps.length > 3) {
issues.push('long_chain');
}
// Check https→http downgrade
for (let i = 0; i < chain.length - 1; i++) {
const curr = chain[i];
const next = chain[i + 1];
if (curr.url.startsWith('https://') && next.url.startsWith('http://')) {
issues.push('mixed_protocol');
break;
}
}
// Check 302 used instead of 301
if (chain.some(s => s.status === 302)) {
issues.push('302_not_301');
}
res.json({
chain,
final_url: finalUrl,
total_time: totalTime,
loop_detected: loopDetected,
issues,
});
});
// ── SVG Editor ──────────────────────────────────────────────────────────────────
app.get('/svgeditor', (req, res) => {
res.sendFile(require('path').join(__dirname, 'public', 'svgeditor.html'));
});
app.post('/api/svg-optimize', express.json({ limit: '5mb' }), (req, res) => {
let svg = req.body.svg || '';
const originalLen = svg.length;
// Remove XML declaration
svg = svg.replace(/<\?xml[^?]*\?>/g, '');
// Remove comments
svg = svg.replace(/<!--[\s\S]*?-->/g, '');
// Remove metadata, title, desc
svg = svg.replace(/<metadata[\s\S]*?<\/metadata>/gi, '');
svg = svg.replace(/<title[\s\S]*?<\/title>/gi, '');
svg = svg.replace(/<desc[\s\S]*?<\/desc>/gi, '');
// Remove empty groups
svg = svg.replace(/<g[^>]*>\s*<\/g>/g, '');
// Remove data-* attributes
svg = svg.replace(/\s+data-[a-z-]+="[^"]*"/g, '');
// Round numbers in d attribute to 2 decimals
svg = svg.replace(/\b(\d+\.\d{3,})\b/g, (m) => parseFloat(m).toFixed(2));
// Collapse whitespace
svg = svg.replace(/\s{2,}/g, ' ');
svg = svg.replace(/>\s+</g, '><');
svg = svg.trim();
const saved = originalLen > 0 ? Math.round((1 - svg.length / originalLen) * 100) : 0;
res.json({ svg, saved: Math.max(0, saved) });
});
app.post('/api/svg-ai', express.json(), (req, res) => {
const { keyword, style, size, color } = req.body;
const icons = {
home: 'M3 12l2-2m0 0l7-7 7 7M5 10v10a1 1 0 001 1h3m10-11l2 2m-2-2v10a1 1 0 01-1 1h-3m-4 0v-5a1 1 0 011-1h2a1 1 0 011 1v5m-4 0h4',
user: 'M16 7a4 4 0 11-8 0 4 4 0 018 0zM12 14a7 7 0 00-7 7h14a7 7 0 00-7-7z',
search: 'M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z',
heart: 'M21 8.25c0-2.485-2.099-4.5-4.688-4.5-1.935 0-3.597 1.126-4.312 2.733-.715-1.607-2.377-2.733-4.313-2.733C5.1 3.75 3 5.765 3 8.25c0 7.22 9 12 9 12s9-4.78 9-12z',
star: 'M11.48 3.499a.562.562 0 011.04 0l2.125 5.111a.563.563 0 00.475.345l5.518.442c.499.04.701.663.321.988l-4.204 3.602a.563.563 0 00-.182.557l1.285 5.385a.562.562 0 01-.84.61l-4.725-2.885a.563.563 0 00-.586 0L6.982 20.54a.562.562 0 01-.84-.61l1.285-5.386a.562.562 0 00-.182-.557l-4.204-3.602a.563.563 0 01.321-.988l5.518-.442a.563.563 0 00.475-.345L11.48 3.5z',
check: 'M4.5 12.75l6 6 9-13.5',
close: 'M6 18L18 6M6 6l12 12',
plus: 'M12 4.5v15m7.5-7.5h-15',
};
const pathD = icons[keyword] || icons['star'];
const s = parseInt(size) || 48;
const c = color || '#0054e6';
let svgStr;
if (style === 'filled') {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="${c}"><path d="${pathD}"/></svg>`;
} else if (style === 'duotone') {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="${c}" fill-opacity="0.2" stroke="${c}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="${pathD}"/></svg>`;
} else {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="${c}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="${pathD}"/></svg>`;
}
res.json({ svg: svgStr, keyword });
});
app.listen(3000, () => console.log('Server on http://localhost:3000'));