Refactor: modular architecture, security, config

- Split monolithic server.js (1111 lines) into route modules
- Add .env config (port, session secret, quality, limits)
- Add SSRF protection for parser/proxy/redirect endpoints
- Add optional password auth middleware
- Add structured logger (replaces raw fs.appendFileSync)
- Add graceful shutdown with timeout
- Add extended /health endpoint (uptime, memory, pid)
- Add ecosystem.config.js for PM2 (memory limit, restart policy)
- Make compress quality configurable (was hardcoded 60)
- Expand SVG AI icons library (7 -> 20 icons)
- Add dotenv dependency
This commit is contained in:
treamz 2026-03-21 22:18:54 +03:00
parent 54d220e0f4
commit 5b528d243a
15 changed files with 1112 additions and 1085 deletions

21
ecosystem.config.js Normal file
View File

@ -0,0 +1,21 @@
module.exports = {
apps: [{
name: 'images',
script: 'server.js',
cwd: '/mnt/webdata/www/images.wadevelop.ru',
instances: 1,
exec_mode: 'fork',
watch: false,
max_memory_restart: '256M',
restart_delay: 3000,
max_restarts: 10,
min_uptime: 5000,
env: {
NODE_ENV: 'production',
},
error_file: '/home/treamz/.pm2/logs/images-error.log',
out_file: '/home/treamz/.pm2/logs/images-out.log',
merge_logs: true,
log_date_format: 'YYYY-MM-DD HH:mm:ss',
}],
};

44
lib/auth.js Normal file
View File

@ -0,0 +1,44 @@
/**
* Optional password auth middleware.
* If AUTH_PASSWORD is set in .env, requires ?key=<password> or session auth.
* Static assets and health check are always public.
*/
const PUBLIC_PATHS = ['/health', '/favicon.ico'];
function authMiddleware(req, res, next) {
const password = process.env.AUTH_PASSWORD;
// No password set — everything is public
if (!password) return next();
// Always allow public paths and static assets
if (PUBLIC_PATHS.includes(req.path)) return next();
if (req.path.startsWith('/vendor/')) return next();
// Check session
if (req.session && req.session.authenticated) return next();
// Check query param
if (req.query.key === password) {
if (req.session) req.session.authenticated = true;
return next();
}
// Show login form for GET requests to pages
if (req.method === 'GET' && !req.path.startsWith('/api/') && !req.path.startsWith('/download/')) {
return res.status(401).send(`<!DOCTYPE html>
<html><head><meta charset="UTF-8"><title>Login</title>
<style>body{font-family:sans-serif;display:flex;justify-content:center;align-items:center;height:100vh;margin:0;background:#060c18;color:#c0c0c0}
form{background:#091020;padding:32px;border-radius:12px;border:1px solid #162040}
input{display:block;margin:8px 0;padding:10px;background:#0a1628;border:1px solid #162040;color:#c0c0c0;border-radius:6px;width:200px}
button{padding:10px 24px;background:#0054e6;color:#fff;border:none;border-radius:6px;cursor:pointer;width:100%}
</style></head><body>
<form method="GET"><label>Пароль</label><input type="password" name="key" autofocus><button>Войти</button></form>
</body></html>`);
}
return res.status(401).json({ error: 'Unauthorized' });
}
module.exports = authMiddleware;

31
lib/logger.js Normal file
View File

@ -0,0 +1,31 @@
const fs = require('fs');
const path = require('path');
const LOG_FILE = path.join(__dirname, '..', 'compress.log');
const LEVEL_PRIORITY = { error: 0, warn: 1, info: 2, debug: 3 };
const currentLevel = LEVEL_PRIORITY[process.env.LOG_LEVEL || 'info'] ?? 2;
function formatTimestamp() {
return new Date().toISOString();
}
function log(level, msg, meta) {
if ((LEVEL_PRIORITY[level] ?? 2) > currentLevel) return;
const ts = formatTimestamp();
const metaStr = meta ? ' ' + JSON.stringify(meta) : '';
const line = `[${ts}] [${level.toUpperCase()}] ${msg}${metaStr}`;
process.stdout.write(line + '\n');
}
function logToFile(text) {
const ts = formatTimestamp();
fs.appendFile(LOG_FILE, `[${ts}] ${text}\n`, () => {});
}
module.exports = {
info: (msg, meta) => log('info', msg, meta),
warn: (msg, meta) => log('warn', msg, meta),
error: (msg, meta) => log('error', msg, meta),
debug: (msg, meta) => log('debug', msg, meta),
logToFile,
};

80
lib/ssrf.js Normal file
View File

@ -0,0 +1,80 @@
const { URL } = require('url');
const dns = require('dns');
const net = require('net');
const BLOCKED_RANGES = [
// IPv4 private
/^10\./,
/^172\.(1[6-9]|2\d|3[01])\./,
/^192\.168\./,
/^127\./,
/^0\./,
// Link-local
/^169\.254\./,
// Loopback IPv6
/^::1$/,
/^fc/i,
/^fd/i,
/^fe80/i,
];
function isPrivateIP(ip) {
return BLOCKED_RANGES.some(re => re.test(ip));
}
/**
* Validate URL is safe for server-side requests (no SSRF)
* Returns { safe: true, url } or { safe: false, error }
*/
async function validateUrl(rawUrl) {
if (!rawUrl || typeof rawUrl !== 'string') {
return { safe: false, error: 'URL обязателен' };
}
let parsed;
try {
parsed = new URL(rawUrl);
} catch {
return { safe: false, error: 'Некорректный URL' };
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return { safe: false, error: 'Поддерживаются только http и https' };
}
const hostname = parsed.hostname;
// Block direct IP access to private ranges
if (net.isIP(hostname)) {
if (isPrivateIP(hostname)) {
return { safe: false, error: 'Доступ к внутренним адресам запрещён' };
}
return { safe: true, url: rawUrl };
}
// Block common internal hostnames
const lower = hostname.toLowerCase();
if (lower === 'localhost' || lower.endsWith('.local') || lower.endsWith('.internal')) {
return { safe: false, error: 'Доступ к внутренним адресам запрещён' };
}
// DNS resolve and check IP
return new Promise((resolve) => {
dns.resolve4(hostname, (err, addresses) => {
if (err || !addresses || addresses.length === 0) {
// Allow through — DNS might fail but fetch will handle it
resolve({ safe: true, url: rawUrl });
return;
}
for (const addr of addresses) {
if (isPrivateIP(addr)) {
resolve({ safe: false, error: 'Доступ к внутренним адресам запрещён' });
return;
}
}
resolve({ safe: true, url: rawUrl });
});
});
}
module.exports = { validateUrl, isPrivateIP };

13
package-lock.json generated
View File

@ -11,6 +11,7 @@
"dependencies": { "dependencies": {
"@mozilla/readability": "^0.6.0", "@mozilla/readability": "^0.6.0",
"archiver": "^7.0.1", "archiver": "^7.0.1",
"dotenv": "^17.3.1",
"express": "^5.1.0", "express": "^5.1.0",
"express-rate-limit": "^8.3.0", "express-rate-limit": "^8.3.0",
"express-session": "^1.18.1", "express-session": "^1.18.1",
@ -1258,6 +1259,18 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/dotenv": {
"version": "17.3.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz",
"integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": { "node_modules/dunder-proto": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",

View File

@ -14,6 +14,7 @@
"dependencies": { "dependencies": {
"@mozilla/readability": "^0.6.0", "@mozilla/readability": "^0.6.0",
"archiver": "^7.0.1", "archiver": "^7.0.1",
"dotenv": "^17.3.1",
"express": "^5.1.0", "express": "^5.1.0",
"express-rate-limit": "^8.3.0", "express-rate-limit": "^8.3.0",
"express-session": "^1.18.1", "express-session": "^1.18.1",

210
routes/compress.js Normal file
View File

@ -0,0 +1,210 @@
const express = require('express');
const multer = require('multer');
const sharp = require('sharp');
const fs = require('fs');
const path = require('path');
const archiver = require('archiver');
const rateLimit = require('express-rate-limit');
const geoip = require('geoip-lite');
const log = require('../lib/logger');
const router = express.Router();
const UPLOADS_DIR = path.join(__dirname, '..', 'uploads');
const DOWNLOADS_DIR = path.join(__dirname, '..', 'downloads');
for (const dir of [UPLOADS_DIR, DOWNLOADS_DIR]) {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}
// Cleanup old files
function cleanupDir(dir, maxAgeMs) {
try {
const files = fs.readdirSync(dir);
const now = Date.now();
for (const file of files) {
const filePath = path.join(dir, file);
try {
const stat = fs.statSync(filePath);
if (now - stat.mtimeMs > maxAgeMs) fs.unlinkSync(filePath);
} catch {}
}
} catch {}
}
setInterval(() => cleanupDir(UPLOADS_DIR, 10 * 60 * 1000), 2 * 60 * 1000);
setInterval(() => cleanupDir(DOWNLOADS_DIR, 30 * 60 * 1000), 5 * 60 * 1000);
// Multer config
const maxFileSize = (parseInt(process.env.MAX_FILE_SIZE_MB) || 20) * 1024 * 1024;
const maxFiles = parseInt(process.env.MAX_FILES) || 50;
const upload = multer({
dest: UPLOADS_DIR,
limits: { fileSize: maxFileSize },
fileFilter: (req, file, cb) => {
const validTypes = ['image/jpeg', 'image/png', 'image/webp'];
if (validTypes.includes(file.mimetype)) cb(null, true);
else cb(new Error('Unsupported file type'));
},
});
// Rate limiter
const limiter = rateLimit({
windowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS) || 60000,
max: parseInt(process.env.RATE_LIMIT_MAX) || 30,
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Слишком много запросов. Попробуйте через минуту.' },
});
// Transliteration
const TRANSLIT_MAP = {
'а':'a','б':'b','в':'v','г':'g','д':'d','е':'e','ё':'yo','ж':'zh','з':'z','и':'i',
'й':'y','к':'k','л':'l','м':'m','н':'n','о':'o','п':'p','р':'r','с':'s','т':'t',
'у':'u','ф':'f','х':'kh','ц':'ts','ч':'ch','ш':'sh','щ':'shch','ъ':'','ы':'y',
'ь':'','э':'e','ю':'yu','я':'ya',
'А':'A','Б':'B','В':'V','Г':'G','Д':'D','Е':'E','Ё':'Yo','Ж':'Zh','З':'Z','И':'I',
'Й':'Y','К':'K','Л':'L','М':'M','Н':'N','О':'O','П':'P','Р':'R','С':'S','Т':'T',
'У':'U','Ф':'F','Х':'Kh','Ц':'Ts','Ч':'Ch','Ш':'Sh','Щ':'Shch','Ъ':'','Ы':'Y',
'Ь':'','Э':'E','Ю':'Yu','Я':'Ya',
};
function fixMulterFilename(str) {
try {
const buf = Buffer.from(str, 'latin1');
const decoded = buf.toString('utf8');
if (/[а-яёА-ЯЁ]/.test(decoded)) return decoded;
} catch {}
return str;
}
function transliterate(str) {
const fixed = fixMulterFilename(str);
return fixed.replace(/[а-яёА-ЯЁ]/g, ch => TRANSLIT_MAP[ch] !== undefined ? TRANSLIT_MAP[ch] : ch)
.replace(/[^\w.\-]/g, '_').replace(/_+/g, '_').replace(/^_|_$/g, '');
}
function getOutputExtension(format, originalName) {
if (format === 'webp') return '.webp';
if (format === 'jpeg') return '.jpg';
if (format === 'png') return '.png';
return path.extname(originalName);
}
function changeExtension(filename, newExt) {
return path.basename(filename, path.extname(filename)) + newExt;
}
function getCountry(ip) {
const clean = ip.replace(/^::ffff:/, '');
if (clean === '127.0.0.1' || clean === '::1') return 'local';
const geo = geoip.lookup(clean);
return geo ? geo.country : '??';
}
// Compress page
router.get('/', (req, res) => {
res.sendFile(path.join(__dirname, '..', 'public', 'compress.html'));
});
// Compress endpoint
router.post('/', limiter, upload.array('images', maxFiles), async (req, res) => {
const resize = parseInt(req.body.resize || '0', 10);
const format = (req.body.format || 'original').toLowerCase();
const quality = Math.min(100, Math.max(1, parseInt(req.body.quality || process.env.COMPRESS_QUALITY || '60', 10)));
const validFormats = ['original', 'webp', 'jpeg', 'png'];
if (!validFormats.includes(format)) {
return res.status(400).json({ error: 'Неверный формат. Допустимые: original, webp, jpeg, png' });
}
if (!req.files || !req.files.length) {
return res.status(400).json({ error: 'Файлы не загружены' });
}
const archiveName = `archive_${Date.now()}.zip`;
const archivePath = path.join(DOWNLOADS_DIR, archiveName);
const output = fs.createWriteStream(archivePath);
const archive = archiver('zip', { zlib: { level: 9 } });
const stats = [];
const country = getCountry(req.ip);
try {
archive.pipe(output);
for (const file of req.files) {
const inputPath = file.path;
const originalSize = file.size;
const image = sharp(inputPath);
const metadata = await image.metadata();
if (resize > 0 && metadata.width && metadata.height && Math.max(metadata.width, metadata.height) > resize) {
const scale = resize / Math.max(metadata.width, metadata.height);
const newWidth = Math.round(metadata.width * scale);
const newHeight = Math.round(metadata.height * scale);
image.resize({ width: newWidth, height: newHeight });
const msg = `[${country}] ${req.ip} resized: ${file.originalname} ${metadata.width}x${metadata.height}${newWidth}x${newHeight} (${(originalSize / 1024).toFixed(0)}KB)`;
log.info(msg);
log.logToFile(msg);
} else {
const msg = `[${country}] ${req.ip} compress: ${file.originalname} ${metadata.width}x${metadata.height} format=${format} q=${quality} (${(originalSize / 1024).toFixed(0)}KB)`;
log.info(msg);
log.logToFile(msg);
}
let buffer;
if (format === 'webp') {
buffer = await image.webp({ quality }).toBuffer();
} else if (format === 'jpeg') {
buffer = await image.jpeg({ quality }).toBuffer();
} else if (format === 'png') {
buffer = await image.png({ compressionLevel: 9 }).toBuffer();
} else {
if (file.mimetype === 'image/jpeg') buffer = await image.jpeg({ quality }).toBuffer();
else if (file.mimetype === 'image/png') buffer = await image.png({ compressionLevel: 9 }).toBuffer();
else if (file.mimetype === 'image/webp') buffer = await image.webp({ quality }).toBuffer();
else continue;
}
const readableName = fixMulterFilename(file.originalname);
const newExt = getOutputExtension(format, readableName);
const rawName = format === 'original' ? readableName : changeExtension(readableName, newExt);
const outputName = transliterate(rawName);
archive.append(buffer, { name: outputName });
const compressedSize = buffer.length;
const savings = originalSize > 0 ? Math.round((1 - compressedSize / originalSize) * 100) : 0;
stats.push({ filename: readableName, outputFilename: outputName, originalSize, compressedSize, savings });
}
await archive.finalize();
await new Promise((resolve, reject) => { output.on('close', resolve); output.on('error', reject); });
res.json({ success: true, downloadUrl: `/download/${archiveName}`, stats });
} catch (err) {
log.error('Compression error', { error: err.message });
try { fs.existsSync(archivePath) && fs.unlinkSync(archivePath); } catch {}
res.status(500).json({ error: 'Ошибка при сжатии' });
} finally {
if (req.files) {
req.files.forEach(file => {
try { fs.existsSync(file.path) && fs.unlinkSync(file.path); } catch {}
});
}
}
});
// Multer error handler
router.use((err, req, res, next) => {
if (err instanceof multer.MulterError) {
if (err.code === 'LIMIT_FILE_SIZE') return res.status(413).json({ error: `Файл слишком большой. Максимум ${process.env.MAX_FILE_SIZE_MB || 20}MB.` });
if (err.code === 'LIMIT_FILE_COUNT') return res.status(400).json({ error: `Слишком много файлов. Максимум ${maxFiles}.` });
return res.status(400).json({ error: err.message });
}
if (err) return res.status(500).json({ error: 'Внутренняя ошибка сервера' });
next();
});
module.exports = router;
module.exports.DOWNLOADS_DIR = DOWNLOADS_DIR;

100
routes/httpclient.js Normal file
View File

@ -0,0 +1,100 @@
const express = require('express');
const rateLimit = require('express-rate-limit');
const path = require('path');
const { validateUrl } = require('../lib/ssrf');
const router = express.Router();
// History (per session)
router.get('/api/history', (req, res) => {
res.json(req.session.httpHistory || []);
});
router.post('/api/history/add', express.json(), (req, res) => {
if (!req.session.httpHistory) req.session.httpHistory = [];
req.session.httpHistory.unshift(req.body);
if (req.session.httpHistory.length > 30) req.session.httpHistory.length = 30;
res.json({ ok: true });
});
router.delete('/api/history', (req, res) => {
req.session.httpHistory = [];
res.json({ ok: true });
});
// Proxy
const proxyLimiter = rateLimit({
windowMs: 60 * 1000,
max: parseInt(process.env.PROXY_RATE_LIMIT_MAX) || 60,
message: { error: 'Слишком много запросов. Попробуйте через минуту.' },
});
router.post('/api/proxy', proxyLimiter, express.json({ limit: '10mb' }), async (req, res) => {
const { url, method = 'GET', headers = {}, body, timeout = 30000 } = req.body || {};
const check = await validateUrl(url);
if (!check.safe) return res.status(400).json({ error: check.error });
const startTime = Date.now();
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.min(Number(timeout) || 30000, 60000));
const fetchOptions = {
method: method.toUpperCase(),
headers: Object.fromEntries(
Object.entries(headers).filter(([k]) => !['host', 'connection', 'transfer-encoding'].includes(k.toLowerCase()))
),
signal: controller.signal,
redirect: 'follow',
};
if (body != null && !['GET', 'HEAD'].includes(method.toUpperCase())) {
fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
}
const response = await fetch(check.url, fetchOptions);
clearTimeout(timer);
const elapsed = Date.now() - startTime;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const MAX_SIZE = 10 * 1024 * 1024;
const reader = response.body?.getReader();
const chunks = [];
let totalSize = 0;
let truncated = false;
if (reader) {
while (true) {
const { done, value } = await reader.read();
if (done) break;
totalSize += value.length;
if (totalSize > MAX_SIZE) { truncated = true; break; }
chunks.push(value);
}
}
const buffer = Buffer.concat(chunks.map(c => Buffer.from(c)));
const responseBody = buffer.toString('utf-8');
res.json({
status: response.status, statusText: response.statusText,
headers: responseHeaders, body: responseBody,
time: elapsed, size: totalSize, truncated, url: response.url,
});
} catch (err) {
const elapsed = Date.now() - startTime;
if (err.name === 'AbortError') return res.status(408).json({ error: 'Таймаут запроса', time: elapsed });
res.status(500).json({ error: err.message, time: elapsed });
}
});
// Page
router.get('/httpclient', (req, res) => {
res.sendFile(path.join(__dirname, '..', 'public', 'httpclient.html'));
});
module.exports = router;

68
routes/logs.js Normal file
View File

@ -0,0 +1,68 @@
const express = require('express');
const fs = require('fs');
const path = require('path');
const router = express.Router();
const LOG_FILE = path.join(__dirname, '..', 'compress.log');
// JSON API
router.get('/api/logs', (req, res) => {
const lines = parseInt(req.query.lines || '100', 10);
const filter = (req.query.filter || '').toLowerCase();
let logLines = [];
try {
const content = fs.readFileSync(LOG_FILE, 'utf8');
logLines = content.trim().split('\n').filter(Boolean);
} catch {}
if (filter) logLines = logLines.filter(l => l.toLowerCase().includes(filter));
logLines = logLines.slice(-Math.min(lines, 1000)).reverse();
res.json({ lines: logLines, total: logLines.length });
});
// Server-rendered logs page
router.get('/logs', (req, res) => {
const lines = parseInt(req.query.lines || '100', 10);
const filter = (req.query.filter || '').toLowerCase();
let logLines = [];
try {
const content = fs.readFileSync(LOG_FILE, 'utf8');
logLines = content.trim().split('\n').filter(Boolean);
} catch {}
if (filter) logLines = logLines.filter(l => l.toLowerCase().includes(filter));
logLines = logLines.slice(-Math.min(lines, 1000)).reverse();
const rendered = logLines.map(line => {
const escaped = line.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
return `<div class="log-line">${escaped}</div>`;
}).join('\n');
res.send(`<!DOCTYPE html>
<html lang="ru" class="dark"><head>
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Logs</title>
<style>
*{margin:0;padding:0;box-sizing:border-box}
body{font-family:'JetBrains Mono',monospace;font-size:13px;background:#060c18;color:#c0c0c0;padding:16px}
.header{display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:16px}
.header h1{font-size:16px;color:#fff;font-weight:600}
.header input{background:#091020;border:1px solid #162040;color:#c0c0c0;padding:6px 10px;border-radius:6px;font-size:12px}
.log-line{padding:6px 10px;border-bottom:1px solid #091020;line-height:1.5;white-space:pre-wrap;word-break:break-all}
.log-line:hover{background:#091020}
.empty{text-align:center;padding:40px;color:#555}
a{color:#0054e6;text-decoration:none}
</style></head><body>
<div class="header">
<a href="/">&larr;</a>
<h1>Logs (${logLines.length})</h1>
<form><input name="filter" placeholder="Filter..." value="${filter.replace(/"/g, '&quot;')}">
<input name="lines" type="number" value="${lines}" style="width:70px"></form>
</div>
${logLines.length ? rendered : '<div class="empty">No logs</div>'}
</body></html>`);
});
module.exports = router;

16
routes/pages.js Normal file
View File

@ -0,0 +1,16 @@
const express = require('express');
const path = require('path');
const router = express.Router();
const pub = (...p) => path.join(__dirname, '..', 'public', ...p);
router.get('/', (req, res) => res.sendFile(pub('index.html')));
router.get('/home', (req, res) => res.sendFile(pub('index.html')));
router.get('/md', (req, res) => res.sendFile(pub('md.html')));
router.get('/editor', (req, res) => res.sendFile(pub('editor.html')));
router.get('/sanitizer', (req, res) => res.sendFile(pub('sanitizer.html')));
router.get('/converter', (req, res) => res.sendFile(pub('converter.html')));
router.get('/formatter', (req, res) => res.sendFile(pub('formatter.html')));
router.get('/password', (req, res) => res.sendFile(pub('password.html')));
module.exports = router;

228
routes/parser.js Normal file
View File

@ -0,0 +1,228 @@
const express = require('express');
const { JSDOM } = require('jsdom');
const { Readability } = require('@mozilla/readability');
const iconv = require('iconv-lite');
const geoip = require('geoip-lite');
const log = require('../lib/logger');
const { validateUrl } = require('../lib/ssrf');
const router = express.Router();
// In-memory cache (max 50 entries, 10 min TTL)
const cache = new Map();
const CACHE_TTL = 10 * 60 * 1000;
const CACHE_MAX = 50;
function getCached(url) {
const entry = cache.get(url);
if (!entry) return null;
if (Date.now() - entry.ts > CACHE_TTL) { cache.delete(url); return null; }
return entry.data;
}
function setCache(url, data) {
if (cache.size >= CACHE_MAX) {
const oldest = cache.keys().next().value;
cache.delete(oldest);
}
cache.set(url, { data, ts: Date.now() });
}
function getCountry(ip) {
const clean = ip.replace(/^::ffff:/, '');
if (clean === '127.0.0.1' || clean === '::1') return 'local';
const geo = geoip.lookup(clean);
return geo ? geo.country : '??';
}
function detectCharset(contentType, htmlBuffer) {
if (contentType) {
const match = contentType.match(/charset=([^\s;]+)/i);
if (match) return match[1].trim().toLowerCase();
}
const head = htmlBuffer.slice(0, 4096).toString('ascii');
const metaMatch = head.match(/charset=["']?([^"'\s;>]+)/i);
if (metaMatch) return metaMatch[1].trim().toLowerCase();
return 'utf-8';
}
async function fetchAndParse(url) {
const cached = getCached(url);
if (cached) return cached;
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 10000);
let response;
try {
response = await fetch(url, {
signal: controller.signal,
redirect: 'follow',
follow: 5,
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WAParser/1.0)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru,en;q=0.5',
},
});
} finally {
clearTimeout(timeout);
}
if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
const contentType = response.headers.get('content-type') || '';
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
throw new Error('URL не содержит HTML-страницу');
}
const contentLength = parseInt(response.headers.get('content-length') || '0', 10);
if (contentLength > 2 * 1024 * 1024) throw new Error('Страница слишком большая (>2MB)');
const arrayBuf = await response.arrayBuffer();
const htmlBuffer = Buffer.from(arrayBuf);
if (htmlBuffer.length > 2 * 1024 * 1024) throw new Error('Страница слишком большая (>2MB)');
const charset = detectCharset(contentType, htmlBuffer);
let html;
if (charset === 'utf-8' || charset === 'utf8') html = htmlBuffer.toString('utf-8');
else if (iconv.encodingExists(charset)) html = iconv.decode(htmlBuffer, charset);
else html = htmlBuffer.toString('utf-8');
const dom = new JSDOM(html, { url });
const doc = dom.window.document;
const getMeta = (name) => {
const el = doc.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
return el ? el.getAttribute('content') : null;
};
const metadata = {
og_title: getMeta('og:title'),
og_description: getMeta('og:description'),
og_image: getMeta('og:image'),
og_site_name: getMeta('og:site_name'),
description: getMeta('description'),
author: getMeta('author') || getMeta('article:author'),
date: getMeta('article:published_time') || getMeta('date') || getMeta('pubdate'),
};
const reader = new Readability(doc);
const article = reader.parse();
let images = [];
let links = [];
if (article && article.content) {
const contentDom = new JSDOM(article.content);
const contentDoc = contentDom.window.document;
contentDoc.querySelectorAll('img').forEach(img => {
const src = img.getAttribute('src') || img.getAttribute('data-src');
if (src) { try { images.push(new URL(src, url).href); } catch {} }
});
contentDoc.querySelectorAll('a[href]').forEach(a => {
const href = a.getAttribute('href');
if (href && href.startsWith('http')) links.push({ text: a.textContent.trim(), href });
});
contentDom.window.close();
}
let cleanHtml = article ? article.content : '';
if (cleanHtml) {
const cleanDom = new JSDOM(cleanHtml);
const cleanDoc = cleanDom.window.document;
cleanDoc.querySelectorAll('script, style, iframe, object, embed').forEach(el => el.remove());
cleanDoc.querySelectorAll('*').forEach(el => {
Array.from(el.attributes).forEach(attr => {
if (attr.name.startsWith('on') || attr.name.startsWith('data-')) el.removeAttribute(attr.name);
});
});
cleanHtml = cleanDoc.body.innerHTML;
cleanDom.window.close();
}
const contentText = article ? article.textContent : '';
const wordCount = contentText ? contentText.trim().split(/\s+/).filter(Boolean).length : 0;
const result = {
url,
title: article ? article.title : (metadata.og_title || doc.title || ''),
author: metadata.author || (article ? article.byline : null),
site: metadata.og_site_name || new URL(url).hostname,
date_published: metadata.date || null,
excerpt: article ? article.excerpt : (metadata.og_description || metadata.description || ''),
content_html: cleanHtml,
content_text: contentText,
lead_image: metadata.og_image || (images.length > 0 ? images[0] : null),
images: [...new Set(images)],
links,
word_count: wordCount,
lang: article ? article.lang : (doc.documentElement.lang || null),
};
dom.window.close();
setCache(url, result);
return result;
}
// Full parse
router.get('/parse', async (req, res) => {
const check = await validateUrl(req.query.url);
if (!check.safe) return res.status(400).json({ error: check.error });
try {
const result = await fetchAndParse(check.url);
const country = getCountry(req.ip);
const msg = `[${country}] ${req.ip} parse: ${check.url} → "${result.title}" (${result.word_count} words)`;
log.info(msg);
log.logToFile(msg);
res.json(result);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка парсинга' });
}
});
// Metadata only
router.get('/metadata', async (req, res) => {
const check = await validateUrl(req.query.url);
if (!check.safe) return res.status(400).json({ error: check.error });
try {
const result = await fetchAndParse(check.url);
res.json({
url: result.url, title: result.title, description: result.excerpt,
image: result.lead_image, site: result.site, author: result.author,
date_published: result.date_published, lang: result.lang,
});
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Text only
router.get('/text', async (req, res) => {
const check = await validateUrl(req.query.url);
if (!check.safe) return res.status(400).json({ error: check.error });
try {
const result = await fetchAndParse(check.url);
res.type('text/plain').send(result.content_text);
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Preview card
router.get('/preview', async (req, res) => {
const check = await validateUrl(req.query.url);
if (!check.safe) return res.status(400).json({ error: check.error });
try {
const result = await fetchAndParse(check.url);
res.json({ url: result.url, title: result.title, description: result.excerpt, image: result.lead_image, site: result.site });
} catch (err) {
res.status(500).json({ error: err.message || 'Ошибка' });
}
});
// Parser page
router.get('/parser', (req, res) => {
res.sendFile(require('path').join(__dirname, '..', 'public', 'parser.html'));
});
module.exports = router;

62
routes/placeholder.js Normal file
View File

@ -0,0 +1,62 @@
const express = require('express');
const sharp = require('sharp');
const path = require('path');
const log = require('../lib/logger');
const router = express.Router();
function normalizeColor(color) {
if (!color) return '#cccccc';
if (/^[0-9A-Fa-f]{6}$/.test(color)) return '#' + color;
if (/^[0-9A-Fa-f]{3}$/.test(color)) return '#' + color;
return color.toLowerCase();
}
function escapeXml(str) {
return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
}
async function placeholderHandler(req, res) {
const { size, bg, fg, text: textFromPath } = req.params;
const { text: textFromQuery, format = 'png', fontsize } = req.query;
const text = textFromQuery || textFromPath || '';
const [w, h] = size.split('x').map(Number);
const width = Math.min(Math.max(w || 300, 1), 4000);
const height = Math.min(Math.max(h || 200, 1), 4000);
const bgColor = normalizeColor(bg);
const fgColor = normalizeColor(fg);
const fontSize = fontsize ? parseInt(fontsize, 10) : Math.max(12, Math.floor(Math.min(width, height) / 8));
const svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">
<rect width="100%" height="100%" fill="${escapeXml(bgColor)}"/>
<text x="50%" y="50%" dominant-baseline="central" text-anchor="middle"
font-family="sans-serif" font-weight="bold" font-size="${fontSize}"
fill="${escapeXml(fgColor)}">${escapeXml(text)}</text>
</svg>`;
try {
let img = sharp(Buffer.from(svg));
if (format === 'jpg' || format === 'jpeg') {
res.type('jpeg').end(await img.jpeg({ quality: 90 }).toBuffer());
} else if (format === 'webp') {
res.type('webp').end(await img.webp({ quality: 90 }).toBuffer());
} else {
res.type('png').end(await img.png().toBuffer());
}
} catch (err) {
log.error('Placeholder error', { error: err.message });
res.status(500).json({ error: 'Generation failed' });
}
}
// Placeholder page
router.get('/', (req, res) => {
res.sendFile(path.join(__dirname, '..', 'public', 'placeholder.html'));
});
router.get('/:size/:bg/:fg', placeholderHandler);
router.get('/:size/:bg/:fg/:text', placeholderHandler);
module.exports = router;

90
routes/redirects.js Normal file
View File

@ -0,0 +1,90 @@
const express = require('express');
const rateLimit = require('express-rate-limit');
const path = require('path');
const { validateUrl } = require('../lib/ssrf');
const router = express.Router();
const UA_STRINGS = {
desktop: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
mobile: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
};
const redirectLimiter = rateLimit({ windowMs: 60_000, max: 20 });
router.post('/api/redirect-analyze', redirectLimiter, express.json(), async (req, res) => {
const { url: rawUrl, userAgent = 'desktop', method = 'GET' } = req.body || {};
const check = await validateUrl(rawUrl);
if (!check.safe) return res.status(400).json({ error: check.error });
const uaString = UA_STRINGS[userAgent] || UA_STRINGS.desktop;
const MAX_STEPS = 15;
const STEP_TIMEOUT = 10_000;
const chain = [];
const visitedUrls = new Set();
let currentUrl = check.url;
let loopDetected = false;
const totalStart = Date.now();
for (let step = 0; step < MAX_STEPS; step++) {
if (visitedUrls.has(currentUrl)) { loopDetected = true; break; }
visitedUrls.add(currentUrl);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), STEP_TIMEOUT);
const stepStart = Date.now();
try {
const response = await fetch(currentUrl, {
method: method.toUpperCase(),
redirect: 'manual',
signal: controller.signal,
headers: {
'User-Agent': uaString,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
},
});
clearTimeout(timer);
const stepTime = Date.now() - stepStart;
const responseHeaders = {};
response.headers.forEach((value, key) => { responseHeaders[key] = value; });
const location = response.headers.get('location') || null;
chain.push({ url: currentUrl, status: response.status, statusText: response.statusText, location, time: stepTime, headers: responseHeaders });
if (response.status < 300 || response.status >= 400) break;
if (!location) break;
try { currentUrl = new URL(location, currentUrl).href; } catch { break; }
} catch (err) {
clearTimeout(timer);
chain.push({ url: currentUrl, status: 0, statusText: err.name === 'AbortError' ? 'Timeout' : err.message, location: null, time: Date.now() - stepStart, headers: {} });
break;
}
}
const totalTime = Date.now() - totalStart;
const finalUrl = chain.length > 0 ? chain[chain.length - 1].url : check.url;
const issues = [];
const redirectSteps = chain.filter(s => s.status >= 300 && s.status < 400);
if (redirectSteps.length > 3) issues.push('long_chain');
for (let i = 0; i < chain.length - 1; i++) {
if (chain[i].url.startsWith('https://') && chain[i + 1].url.startsWith('http://')) { issues.push('mixed_protocol'); break; }
}
if (chain.some(s => s.status === 302)) issues.push('302_not_301');
res.json({ chain, final_url: finalUrl, total_time: totalTime, loop_detected: loopDetected, issues });
});
// Page
router.get('/redirects', (req, res) => {
res.sendFile(path.join(__dirname, '..', 'public', 'redirects.html'));
});
module.exports = router;

76
routes/svgeditor.js Normal file
View File

@ -0,0 +1,76 @@
const express = require('express');
const path = require('path');
const router = express.Router();
// SVG Editor page
router.get('/svgeditor', (req, res) => {
res.sendFile(path.join(__dirname, '..', 'public', 'svgeditor.html'));
});
// SVG Optimize
router.post('/api/svg-optimize', express.json({ limit: '5mb' }), (req, res) => {
let svg = req.body.svg || '';
const originalLen = svg.length;
svg = svg.replace(/<\?xml[^?]*\?>/g, '');
svg = svg.replace(/<!--[\s\S]*?-->/g, '');
svg = svg.replace(/<metadata[\s\S]*?<\/metadata>/gi, '');
svg = svg.replace(/<title[\s\S]*?<\/title>/gi, '');
svg = svg.replace(/<desc[\s\S]*?<\/desc>/gi, '');
svg = svg.replace(/<g[^>]*>\s*<\/g>/g, '');
svg = svg.replace(/\s+data-[a-z-]+="[^"]*"/g, '');
svg = svg.replace(/\b(\d+\.\d{3,})\b/g, (m) => parseFloat(m).toFixed(2));
svg = svg.replace(/\s{2,}/g, ' ');
svg = svg.replace(/>\s+</g, '><');
svg = svg.trim();
const saved = originalLen > 0 ? Math.round((1 - svg.length / originalLen) * 100) : 0;
res.json({ svg, saved: Math.max(0, saved) });
});
// SVG AI — generates simple icons from keywords
const ICON_PATHS = {
home: 'M3 12l2-2m0 0l7-7 7 7M5 10v10a1 1 0 001 1h3m10-11l2 2m-2-2v10a1 1 0 01-1 1h-3m-4 0v-5a1 1 0 011-1h2a1 1 0 011 1v5m-4 0h4',
user: 'M16 7a4 4 0 11-8 0 4 4 0 018 0zM12 14a7 7 0 00-7 7h14a7 7 0 00-7-7z',
search: 'M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z',
heart: 'M21 8.25c0-2.485-2.099-4.5-4.688-4.5-1.935 0-3.597 1.126-4.312 2.733-.715-1.607-2.377-2.733-4.313-2.733C5.1 3.75 3 5.765 3 8.25c0 7.22 9 12 9 12s9-4.78 9-12z',
star: 'M11.48 3.499a.562.562 0 011.04 0l2.125 5.111a.563.563 0 00.475.345l5.518.442c.499.04.701.663.321.988l-4.204 3.602a.563.563 0 00-.182.557l1.285 5.385a.562.562 0 01-.84.61l-4.725-2.885a.563.563 0 00-.586 0L6.982 20.54a.562.562 0 01-.84-.61l1.285-5.386a.562.562 0 00-.182-.557l-4.204-3.602a.563.563 0 01.321-.988l5.518-.442a.563.563 0 00.475-.345L11.48 3.5z',
check: 'M4.5 12.75l6 6 9-13.5',
close: 'M6 18L18 6M6 6l12 12',
plus: 'M12 4.5v15m7.5-7.5h-15',
mail: 'M21.75 6.75v10.5a2.25 2.25 0 01-2.25 2.25h-15a2.25 2.25 0 01-2.25-2.25V6.75m19.5 0A2.25 2.25 0 0019.5 4.5h-15a2.25 2.25 0 00-2.25 2.25m19.5 0v.243a2.25 2.25 0 01-1.07 1.916l-7.5 4.615a2.25 2.25 0 01-2.36 0L3.32 8.91a2.25 2.25 0 01-1.07-1.916V6.75',
phone: 'M2.25 6.75c0 8.284 6.716 15 15 15h2.25a2.25 2.25 0 002.25-2.25v-1.372c0-.516-.351-.966-.852-1.091l-4.423-1.106c-.44-.11-.902.055-1.173.417l-.97 1.293c-.282.376-.769.542-1.21.38a12.035 12.035 0 01-7.143-7.143c-.162-.441.004-.928.38-1.21l1.293-.97c.363-.271.527-.734.417-1.173L6.963 3.102a1.125 1.125 0 00-1.091-.852H4.5A2.25 2.25 0 002.25 4.5v2.25z',
settings: 'M9.594 3.94c.09-.542.56-.94 1.11-.94h2.593c.55 0 1.02.398 1.11.94l.213 1.281c.063.374.313.686.645.87.074.04.147.083.22.127.324.196.72.257 1.075.124l1.217-.456a1.125 1.125 0 011.37.49l1.296 2.247a1.125 1.125 0 01-.26 1.431l-1.003.827c-.293.24-.438.613-.431.992a6.759 6.759 0 010 .255c-.007.378.138.75.43.99l1.005.828c.424.35.534.954.26 1.43l-1.298 2.247a1.125 1.125 0 01-1.369.491l-1.217-.456c-.355-.133-.75-.072-1.076.124a6.57 6.57 0 01-.22.128c-.331.183-.581.495-.644.869l-.213 1.28c-.09.543-.56.941-1.11.941h-2.594c-.55 0-1.02-.398-1.11-.94l-.213-1.281c-.062-.374-.312-.686-.644-.87a6.52 6.52 0 01-.22-.127c-.325-.196-.72-.257-1.076-.124l-1.217.456a1.125 1.125 0 01-1.369-.49l-1.297-2.247a1.125 1.125 0 01.26-1.431l1.004-.827c.292-.24.437-.613.43-.992a6.932 6.932 0 010-.255c.007-.378-.138-.75-.43-.99l-1.004-.828a1.125 1.125 0 01-.26-1.43l1.297-2.247a1.125 1.125 0 011.37-.491l1.216.456c.356.133.751.072 1.076-.124.072-.044.146-.087.22-.128.332-.183.582-.495.644-.869l.214-1.281z M15 12a3 3 0 11-6 0 3 3 0 016 0z',
cart: 'M2.25 3h1.386c.51 0 .955.343 1.087.835l.383 1.437M7.5 14.25a3 3 0 00-3 3h15.75m-12.75-3h11.218c1.121-2.3 2.1-4.684 2.924-7.138a60.114 60.114 0 00-16.536-1.84M7.5 14.25L5.106 5.272M6 20.25a.75.75 0 11-1.5 0 .75.75 0 011.5 0zm12.75 0a.75.75 0 11-1.5 0 .75.75 0 011.5 0z',
lock: 'M16.5 10.5V6.75a4.5 4.5 0 10-9 0v3.75m-.75 11.25h10.5a2.25 2.25 0 002.25-2.25v-6.75a2.25 2.25 0 00-2.25-2.25H6.75a2.25 2.25 0 00-2.25 2.25v6.75a2.25 2.25 0 002.25 2.25z',
eye: 'M2.036 12.322a1.012 1.012 0 010-.639C3.423 7.51 7.36 4.5 12 4.5c4.638 0 8.573 3.007 9.963 7.178.07.207.07.431 0 .639C20.577 16.49 16.64 19.5 12 19.5c-4.638 0-8.573-3.007-9.963-7.178z M15 12a3 3 0 11-6 0 3 3 0 016 0z',
download: 'M3 16.5v2.25A2.25 2.25 0 005.25 21h13.5A2.25 2.25 0 0021 18.75V16.5M16.5 12L12 16.5m0 0L7.5 12m4.5 4.5V3',
upload: 'M3 16.5v2.25A2.25 2.25 0 005.25 21h13.5A2.25 2.25 0 0021 18.75V16.5m-13.5-9L12 3m0 0l4.5 4.5M12 3v13.5',
trash: 'M14.74 9l-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 01-2.244 2.077H8.084a2.25 2.25 0 01-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 00-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 013.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 00-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 00-7.5 0',
edit: 'M16.862 4.487l1.687-1.688a1.875 1.875 0 112.652 2.652L10.582 16.07a4.5 4.5 0 01-1.897 1.13L6 18l.8-2.685a4.5 4.5 0 011.13-1.897l8.932-8.931zm0 0L19.5 7.125M18 14v4.75A2.25 2.25 0 0115.75 21H5.25A2.25 2.25 0 013 18.75V8.25A2.25 2.25 0 015.25 6H10',
arrow_right: 'M13.5 4.5L21 12m0 0l-7.5 7.5M21 12H3',
arrow_left: 'M10.5 19.5L3 12m0 0l7.5-7.5M3 12h18',
menu: 'M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5',
bell: 'M14.857 17.082a23.848 23.848 0 005.454-1.31A8.967 8.967 0 0118 9.75v-.7V9A6 6 0 006 9v.75a8.967 8.967 0 01-2.312 6.022c1.733.64 3.56 1.085 5.455 1.31m5.714 0a24.255 24.255 0 01-5.714 0m5.714 0a3 3 0 11-5.714 0',
};
router.post('/api/svg-ai', express.json(), (req, res) => {
const { keyword, style, size, color } = req.body;
const k = (keyword || '').toLowerCase().trim();
const pathD = ICON_PATHS[k] || ICON_PATHS['star'];
const s = parseInt(size) || 48;
const c = color || '#0054e6';
let svgStr;
if (style === 'filled') {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="${c}"><path d="${pathD}"/></svg>`;
} else if (style === 'duotone') {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="${c}" fill-opacity="0.2" stroke="${c}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="${pathD}"/></svg>`;
} else {
svgStr = `<svg xmlns="http://www.w3.org/2000/svg" width="${s}" height="${s}" viewBox="0 0 24 24" fill="none" stroke="${c}" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="${pathD}"/></svg>`;
}
res.json({ svg: svgStr, keyword: k, available: Object.keys(ICON_PATHS) });
});
module.exports = router;

1145
server.js

File diff suppressed because it is too large Load Diff