From 7387cb98076ce46e6a04a173bdee0b88bc112341 Mon Sep 17 00:00:00 2001 From: TheFlow Date: Tue, 14 Oct 2025 15:58:48 +1300 Subject: [PATCH] security: implement file upload security with ClamAV integration (inst_041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1: File Security Complete ✅ Created file-security.middleware.js with multi-layer validation ✅ Installed multer for file uploads ✅ Created quarantine directories on production and dev ✅ Integrated ClamAV malware scanning Features: - Magic number validation (prevents MIME spoofing) - ClamAV malware scanning (8.7M signatures) - Automatic file quarantine with metadata - Size limits: 10MB documents, 50MB media - MIME type whitelist enforcement - Comprehensive security event logging Middleware provides: - createSecureUpload() - Full pipeline (multer + security) - createFileSecurityMiddleware() - Validation only - Quarantine system with JSON metadata Implements: inst_041 (file upload validation) Refs: docs/plans/security-implementation-roadmap.md Phase 2-P2-2 ClamAV Status: - Version: 1.4.3 - Signatures: 8,724,466 - Daemon: Running (521MB RAM) - Test: EICAR detection confirmed --- package-lock.json | 96 ++++- package.json | 1 + src/middleware/file-security.middleware.js | 400 +++++++++++++++++++++ 3 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 src/middleware/file-security.middleware.js diff --git a/package-lock.json b/package-lock.json index f4fffc26..3bc09e50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "marked": "^11.0.0", "mongodb": "^6.3.0", "mongoose": "^8.19.1", + "multer": "^2.0.2", "puppeteer": "^24.23.0", "sanitize-html": "^2.11.0", "stripe": "^14.25.0", @@ -1751,6 +1752,12 @@ "node": ">= 8" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/aproba": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", @@ -2295,9 +2302,19 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, "license": "MIT" }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2656,6 +2673,21 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/concurrently": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.1.tgz", @@ -5969,6 +6001,15 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", @@ -6132,6 +6173,36 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", + "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", + "object-assign": "^4.1.1", + "type-is": "^1.6.18", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">= 10.16.0" + } + }, + "node_modules/multer/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/mustache": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", @@ -8010,6 +8081,14 @@ "node": ">= 0.8" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/streamx": { "version": "2.23.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", @@ -8610,6 +8689,12 @@ "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", "license": "MIT" }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/uid-safe": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", @@ -8926,6 +9011,15 @@ } } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index dfe99a5c..391ee1f4 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "marked": "^11.0.0", "mongodb": "^6.3.0", "mongoose": "^8.19.1", + "multer": "^2.0.2", "puppeteer": "^24.23.0", "sanitize-html": "^2.11.0", "stripe": "^14.25.0", diff --git a/src/middleware/file-security.middleware.js b/src/middleware/file-security.middleware.js new file mode 100644 index 00000000..8551249e --- /dev/null +++ b/src/middleware/file-security.middleware.js @@ -0,0 +1,400 @@ +/** + * File Security Middleware (inst_041 Implementation) + * + * Multi-layer file upload validation: + * 1. File type validation (magic number check) + * 2. ClamAV malware scanning + * 3. Size limits enforcement + * 4. Quarantine system for suspicious files + * 5. Comprehensive security logging + * + * Reference: docs/plans/security-implementation-roadmap.md Phase 2 + */ + +const path = require('path'); +const fs = require('fs').promises; +const { exec } = require('child_process'); +const { promisify } = require('util'); +const multer = require('multer'); +const { logSecurityEvent, getClientIp } = require('../utils/security-logger'); + +const execAsync = promisify(exec); + +// Configuration +const UPLOAD_DIR = process.env.UPLOAD_DIR || '/tmp/tractatus-uploads'; +const QUARANTINE_DIR = process.env.QUARANTINE_DIR || '/var/quarantine/tractatus'; +const MAX_FILE_SIZE = { + document: 10 * 1024 * 1024, // 10MB + media: 50 * 1024 * 1024, // 50MB + default: 5 * 1024 * 1024 // 5MB +}; + +// Allowed MIME types (whitelist approach) +const ALLOWED_MIME_TYPES = { + document: [ + 'application/pdf', + 'application/msword', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'text/plain', + 'text/markdown' + ], + media: [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + 'video/mp4', + 'video/webm' + ] +}; + +/** + * Ensure upload and quarantine directories exist + */ +async function ensureDirectories() { + try { + await fs.mkdir(UPLOAD_DIR, { recursive: true, mode: 0o750 }); + await fs.mkdir(QUARANTINE_DIR, { recursive: true, mode: 0o750 }); + } catch (error) { + console.error('[FILE SECURITY] Failed to create directories:', error.message); + } +} + +// Initialize directories on module load +ensureDirectories(); + +/** + * Validate file type using magic number (file command) + * Prevents MIME type spoofing + */ +async function validateFileType(filePath, expectedMimeType) { + try { + const { stdout } = await execAsync(`file --mime-type -b "${filePath}"`); + const actualMimeType = stdout.trim(); + + if (actualMimeType !== expectedMimeType) { + return { + valid: false, + actualType: actualMimeType, + expectedType: expectedMimeType, + message: 'File type mismatch (possible MIME spoofing)' + }; + } + + return { valid: true, actualType: actualMimeType }; + } catch (error) { + return { + valid: false, + message: `File type validation failed: ${error.message}` + }; + } +} + +/** + * Scan file with ClamAV + */ +async function scanWithClamAV(filePath) { + try { + await execAsync(`clamdscan --no-summary "${filePath}"`); + return { clean: true, threat: null }; + } catch (error) { + // clamdscan exits with non-zero if virus found + const output = error.stdout || error.stderr || ''; + + if (output.includes('FOUND')) { + const match = output.match(/(.+): (.+) FOUND/); + const threat = match ? match[2] : 'Unknown threat'; + return { clean: false, threat }; + } + + // Other error (daemon not running, etc.) + return { + clean: false, + threat: null, + error: `ClamAV scan failed: ${error.message}` + }; + } +} + +/** + * Move file to quarantine + */ +async function quarantineFile(filePath, reason, metadata) { + try { + const filename = path.basename(filePath); + const timestamp = new Date().toISOString().replace(/:/g, '-'); + const quarantinePath = path.join(QUARANTINE_DIR, `${timestamp}_${filename}`); + + // Move file to quarantine + await fs.rename(filePath, quarantinePath); + + // Create metadata file + const metadataPath = `${quarantinePath}.json`; + await fs.writeFile(metadataPath, JSON.stringify({ + original_path: filePath, + original_name: filename, + quarantine_reason: reason, + quarantine_time: new Date().toISOString(), + ...metadata + }, null, 2)); + + return quarantinePath; + } catch (error) { + console.error('[FILE SECURITY] Quarantine failed:', error.message); + // Try to delete file if quarantine fails + try { + await fs.unlink(filePath); + } catch (unlinkError) { + console.error('[FILE SECURITY] Failed to delete file after quarantine failure:', unlinkError.message); + } + throw error; + } +} + +/** + * File security validation middleware + * Use after multer upload + */ +function createFileSecurityMiddleware(options = {}) { + const { + fileType = 'default', + allowedMimeTypes = ALLOWED_MIME_TYPES.document + } = options; + + return async (req, res, next) => { + // Skip if no file uploaded + if (!req.file && !req.files) { + return next(); + } + + const files = req.files || [req.file]; + const clientIp = getClientIp(req); + const userId = req.user?.id || 'anonymous'; + + try { + for (const file of files) { + if (!file) continue; + + const filePath = file.path; + const filename = file.originalname; + + // 1. Check MIME type is allowed + if (!allowedMimeTypes.includes(file.mimetype)) { + await fs.unlink(filePath); + + await logSecurityEvent({ + type: 'file_upload_rejected', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + filename, + mime_type: file.mimetype, + reason: 'MIME type not allowed' + }, + action: 'rejected', + severity: 'medium' + }); + + return res.status(400).json({ + error: 'Bad Request', + message: `File type ${file.mimetype} is not allowed`, + allowed_types: allowedMimeTypes + }); + } + + // 2. Validate file type with magic number + const typeValidation = await validateFileType(filePath, file.mimetype); + if (!typeValidation.valid) { + await quarantineFile(filePath, 'MIME_TYPE_MISMATCH', { + reported_mime: file.mimetype, + actual_mime: typeValidation.actualType, + user_id: userId, + source_ip: clientIp + }); + + await logSecurityEvent({ + type: 'file_upload_quarantined', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + filename, + reason: 'MIME type mismatch', + reported: file.mimetype, + actual: typeValidation.actualType + }, + action: 'quarantined', + severity: 'high' + }); + + return res.status(403).json({ + error: 'Forbidden', + message: 'File rejected due to security concerns', + code: 'FILE_TYPE_MISMATCH' + }); + } + + // 3. Scan with ClamAV + const scanResult = await scanWithClamAV(filePath); + + if (!scanResult.clean) { + if (scanResult.threat) { + // Malware detected + await quarantineFile(filePath, 'MALWARE_DETECTED', { + threat: scanResult.threat, + user_id: userId, + source_ip: clientIp + }); + + await logSecurityEvent({ + type: 'malware_detected', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + filename, + threat: scanResult.threat, + mime_type: file.mimetype + }, + action: 'quarantined', + severity: 'critical' + }); + + return res.status(403).json({ + error: 'Forbidden', + message: 'File rejected: Security threat detected', + code: 'MALWARE_DETECTED' + }); + } else { + // Scan failed + await logSecurityEvent({ + type: 'file_scan_failed', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + filename, + error: scanResult.error + }, + action: 'blocked', + severity: 'high' + }); + + return res.status(503).json({ + error: 'Service Unavailable', + message: 'File security scan unavailable. Please try again later.', + code: 'SCAN_FAILED' + }); + } + } + + // File passed all checks + await logSecurityEvent({ + type: 'file_upload_validated', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + filename, + mime_type: file.mimetype, + size: file.size + }, + action: 'allowed', + severity: 'low' + }); + } + + // All files passed validation + next(); + + } catch (error) { + console.error('[FILE SECURITY] Validation error:', error); + + await logSecurityEvent({ + type: 'file_validation_error', + sourceIp: clientIp, + userId, + endpoint: req.path, + userAgent: req.get('user-agent'), + details: { + error: error.message + }, + action: 'error', + severity: 'high' + }); + + return res.status(500).json({ + error: 'Internal Server Error', + message: 'File validation failed' + }); + } + }; +} + +/** + * Create multer storage configuration + */ +function createUploadStorage() { + return multer.diskStorage({ + destination: async (req, file, cb) => { + await ensureDirectories(); + cb(null, UPLOAD_DIR); + }, + filename: (req, file, cb) => { + const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1E9)}`; + const ext = path.extname(file.originalname); + const basename = path.basename(file.originalname, ext); + const sanitized = basename.replace(/[^a-zA-Z0-9-_]/g, '_'); + cb(null, `${sanitized}-${uniqueSuffix}${ext}`); + } + }); +} + +/** + * Create multer upload middleware with security + */ +function createSecureUpload(options = {}) { + const { + fileType = 'default', + maxFileSize = MAX_FILE_SIZE.default, + allowedMimeTypes = ALLOWED_MIME_TYPES.document, + fieldName = 'file' + } = options; + + const upload = multer({ + storage: createUploadStorage(), + limits: { + fileSize: maxFileSize, + files: 1 // Single file upload by default + }, + fileFilter: (req, file, cb) => { + // Basic MIME type check (will be verified again with magic number) + if (allowedMimeTypes.includes(file.mimetype)) { + cb(null, true); + } else { + cb(new Error(`File type ${file.mimetype} not allowed`)); + } + } + }); + + return [ + upload.single(fieldName), + createFileSecurityMiddleware({ fileType, allowedMimeTypes }) + ]; +} + +module.exports = { + createFileSecurityMiddleware, + createSecureUpload, + validateFileType, + scanWithClamAV, + quarantineFile, + ALLOWED_MIME_TYPES, + MAX_FILE_SIZE +};