security: implement file upload security with ClamAV integration (inst_041)

Phase 1: File Security Complete
 Created file-security.middleware.js with multi-layer validation
 Installed multer for file uploads
 Created quarantine directories on production and dev
 Integrated ClamAV malware scanning

Features:
- Magic number validation (prevents MIME spoofing)
- ClamAV malware scanning (8.7M signatures)
- Automatic file quarantine with metadata
- Size limits: 10MB documents, 50MB media
- MIME type whitelist enforcement
- Comprehensive security event logging

Middleware provides:
- createSecureUpload() - Full pipeline (multer + security)
- createFileSecurityMiddleware() - Validation only
- Quarantine system with JSON metadata

Implements: inst_041 (file upload validation)
Refs: docs/plans/security-implementation-roadmap.md Phase 2-P2-2

ClamAV Status:
- Version: 1.4.3
- Signatures: 8,724,466
- Daemon: Running (521MB RAM)
- Test: EICAR detection confirmed
This commit is contained in:
TheFlow 2025-10-14 15:58:48 +13:00
parent 142c539717
commit 7387cb9807
3 changed files with 496 additions and 1 deletions

96
package-lock.json generated
View file

@ -22,6 +22,7 @@
"marked": "^11.0.0",
"mongodb": "^6.3.0",
"mongoose": "^8.19.1",
"multer": "^2.0.2",
"puppeteer": "^24.23.0",
"sanitize-html": "^2.11.0",
"stripe": "^14.25.0",
@ -1751,6 +1752,12 @@
"node": ">= 8"
}
},
"node_modules/append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
"license": "MIT"
},
"node_modules/aproba": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz",
@ -2295,9 +2302,19 @@
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
"dev": true,
"license": "MIT"
},
"node_modules/busboy": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
"dependencies": {
"streamsearch": "^1.1.0"
},
"engines": {
"node": ">=10.16.0"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@ -2656,6 +2673,21 @@
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"license": "MIT"
},
"node_modules/concat-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
"engines": [
"node >= 6.0"
],
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.0.2",
"typedarray": "^0.0.6"
}
},
"node_modules/concurrently": {
"version": "9.2.1",
"resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.1.tgz",
@ -5969,6 +6001,15 @@
"node": "*"
}
},
"node_modules/minimist": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/minipass": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
@ -6132,6 +6173,36 @@
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/multer": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
"license": "MIT",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.6.0",
"concat-stream": "^2.0.0",
"mkdirp": "^0.5.6",
"object-assign": "^4.1.1",
"type-is": "^1.6.18",
"xtend": "^4.0.2"
},
"engines": {
"node": ">= 10.16.0"
}
},
"node_modules/multer/node_modules/mkdirp": {
"version": "0.5.6",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
"integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
"license": "MIT",
"dependencies": {
"minimist": "^1.2.6"
},
"bin": {
"mkdirp": "bin/cmd.js"
}
},
"node_modules/mustache": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
@ -8010,6 +8081,14 @@
"node": ">= 0.8"
}
},
"node_modules/streamsearch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/streamx": {
"version": "2.23.0",
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz",
@ -8610,6 +8689,12 @@
"integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==",
"license": "MIT"
},
"node_modules/typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
"license": "MIT"
},
"node_modules/uid-safe": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
@ -8926,6 +9011,15 @@
}
}
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"license": "MIT",
"engines": {
"node": ">=0.4"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

View file

@ -53,6 +53,7 @@
"marked": "^11.0.0",
"mongodb": "^6.3.0",
"mongoose": "^8.19.1",
"multer": "^2.0.2",
"puppeteer": "^24.23.0",
"sanitize-html": "^2.11.0",
"stripe": "^14.25.0",

View file

@ -0,0 +1,400 @@
/**
* File Security Middleware (inst_041 Implementation)
*
* Multi-layer file upload validation:
* 1. File type validation (magic number check)
* 2. ClamAV malware scanning
* 3. Size limits enforcement
* 4. Quarantine system for suspicious files
* 5. Comprehensive security logging
*
* Reference: docs/plans/security-implementation-roadmap.md Phase 2
*/
const path = require('path');
const fs = require('fs').promises;
const { exec } = require('child_process');
const { promisify } = require('util');
const multer = require('multer');
const { logSecurityEvent, getClientIp } = require('../utils/security-logger');
const execAsync = promisify(exec);
// Configuration
const UPLOAD_DIR = process.env.UPLOAD_DIR || '/tmp/tractatus-uploads';
const QUARANTINE_DIR = process.env.QUARANTINE_DIR || '/var/quarantine/tractatus';
const MAX_FILE_SIZE = {
document: 10 * 1024 * 1024, // 10MB
media: 50 * 1024 * 1024, // 50MB
default: 5 * 1024 * 1024 // 5MB
};
// Allowed MIME types (whitelist approach)
const ALLOWED_MIME_TYPES = {
document: [
'application/pdf',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'text/plain',
'text/markdown'
],
media: [
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'video/mp4',
'video/webm'
]
};
/**
* Ensure upload and quarantine directories exist
*/
async function ensureDirectories() {
try {
await fs.mkdir(UPLOAD_DIR, { recursive: true, mode: 0o750 });
await fs.mkdir(QUARANTINE_DIR, { recursive: true, mode: 0o750 });
} catch (error) {
console.error('[FILE SECURITY] Failed to create directories:', error.message);
}
}
// Initialize directories on module load
ensureDirectories();
/**
* Validate file type using magic number (file command)
* Prevents MIME type spoofing
*/
async function validateFileType(filePath, expectedMimeType) {
try {
const { stdout } = await execAsync(`file --mime-type -b "${filePath}"`);
const actualMimeType = stdout.trim();
if (actualMimeType !== expectedMimeType) {
return {
valid: false,
actualType: actualMimeType,
expectedType: expectedMimeType,
message: 'File type mismatch (possible MIME spoofing)'
};
}
return { valid: true, actualType: actualMimeType };
} catch (error) {
return {
valid: false,
message: `File type validation failed: ${error.message}`
};
}
}
/**
* Scan file with ClamAV
*/
async function scanWithClamAV(filePath) {
try {
await execAsync(`clamdscan --no-summary "${filePath}"`);
return { clean: true, threat: null };
} catch (error) {
// clamdscan exits with non-zero if virus found
const output = error.stdout || error.stderr || '';
if (output.includes('FOUND')) {
const match = output.match(/(.+): (.+) FOUND/);
const threat = match ? match[2] : 'Unknown threat';
return { clean: false, threat };
}
// Other error (daemon not running, etc.)
return {
clean: false,
threat: null,
error: `ClamAV scan failed: ${error.message}`
};
}
}
/**
* Move file to quarantine
*/
async function quarantineFile(filePath, reason, metadata) {
try {
const filename = path.basename(filePath);
const timestamp = new Date().toISOString().replace(/:/g, '-');
const quarantinePath = path.join(QUARANTINE_DIR, `${timestamp}_${filename}`);
// Move file to quarantine
await fs.rename(filePath, quarantinePath);
// Create metadata file
const metadataPath = `${quarantinePath}.json`;
await fs.writeFile(metadataPath, JSON.stringify({
original_path: filePath,
original_name: filename,
quarantine_reason: reason,
quarantine_time: new Date().toISOString(),
...metadata
}, null, 2));
return quarantinePath;
} catch (error) {
console.error('[FILE SECURITY] Quarantine failed:', error.message);
// Try to delete file if quarantine fails
try {
await fs.unlink(filePath);
} catch (unlinkError) {
console.error('[FILE SECURITY] Failed to delete file after quarantine failure:', unlinkError.message);
}
throw error;
}
}
/**
* File security validation middleware
* Use after multer upload
*/
function createFileSecurityMiddleware(options = {}) {
const {
fileType = 'default',
allowedMimeTypes = ALLOWED_MIME_TYPES.document
} = options;
return async (req, res, next) => {
// Skip if no file uploaded
if (!req.file && !req.files) {
return next();
}
const files = req.files || [req.file];
const clientIp = getClientIp(req);
const userId = req.user?.id || 'anonymous';
try {
for (const file of files) {
if (!file) continue;
const filePath = file.path;
const filename = file.originalname;
// 1. Check MIME type is allowed
if (!allowedMimeTypes.includes(file.mimetype)) {
await fs.unlink(filePath);
await logSecurityEvent({
type: 'file_upload_rejected',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
filename,
mime_type: file.mimetype,
reason: 'MIME type not allowed'
},
action: 'rejected',
severity: 'medium'
});
return res.status(400).json({
error: 'Bad Request',
message: `File type ${file.mimetype} is not allowed`,
allowed_types: allowedMimeTypes
});
}
// 2. Validate file type with magic number
const typeValidation = await validateFileType(filePath, file.mimetype);
if (!typeValidation.valid) {
await quarantineFile(filePath, 'MIME_TYPE_MISMATCH', {
reported_mime: file.mimetype,
actual_mime: typeValidation.actualType,
user_id: userId,
source_ip: clientIp
});
await logSecurityEvent({
type: 'file_upload_quarantined',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
filename,
reason: 'MIME type mismatch',
reported: file.mimetype,
actual: typeValidation.actualType
},
action: 'quarantined',
severity: 'high'
});
return res.status(403).json({
error: 'Forbidden',
message: 'File rejected due to security concerns',
code: 'FILE_TYPE_MISMATCH'
});
}
// 3. Scan with ClamAV
const scanResult = await scanWithClamAV(filePath);
if (!scanResult.clean) {
if (scanResult.threat) {
// Malware detected
await quarantineFile(filePath, 'MALWARE_DETECTED', {
threat: scanResult.threat,
user_id: userId,
source_ip: clientIp
});
await logSecurityEvent({
type: 'malware_detected',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
filename,
threat: scanResult.threat,
mime_type: file.mimetype
},
action: 'quarantined',
severity: 'critical'
});
return res.status(403).json({
error: 'Forbidden',
message: 'File rejected: Security threat detected',
code: 'MALWARE_DETECTED'
});
} else {
// Scan failed
await logSecurityEvent({
type: 'file_scan_failed',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
filename,
error: scanResult.error
},
action: 'blocked',
severity: 'high'
});
return res.status(503).json({
error: 'Service Unavailable',
message: 'File security scan unavailable. Please try again later.',
code: 'SCAN_FAILED'
});
}
}
// File passed all checks
await logSecurityEvent({
type: 'file_upload_validated',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
filename,
mime_type: file.mimetype,
size: file.size
},
action: 'allowed',
severity: 'low'
});
}
// All files passed validation
next();
} catch (error) {
console.error('[FILE SECURITY] Validation error:', error);
await logSecurityEvent({
type: 'file_validation_error',
sourceIp: clientIp,
userId,
endpoint: req.path,
userAgent: req.get('user-agent'),
details: {
error: error.message
},
action: 'error',
severity: 'high'
});
return res.status(500).json({
error: 'Internal Server Error',
message: 'File validation failed'
});
}
};
}
/**
* Create multer storage configuration
*/
function createUploadStorage() {
return multer.diskStorage({
destination: async (req, file, cb) => {
await ensureDirectories();
cb(null, UPLOAD_DIR);
},
filename: (req, file, cb) => {
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1E9)}`;
const ext = path.extname(file.originalname);
const basename = path.basename(file.originalname, ext);
const sanitized = basename.replace(/[^a-zA-Z0-9-_]/g, '_');
cb(null, `${sanitized}-${uniqueSuffix}${ext}`);
}
});
}
/**
* Create multer upload middleware with security
*/
function createSecureUpload(options = {}) {
const {
fileType = 'default',
maxFileSize = MAX_FILE_SIZE.default,
allowedMimeTypes = ALLOWED_MIME_TYPES.document,
fieldName = 'file'
} = options;
const upload = multer({
storage: createUploadStorage(),
limits: {
fileSize: maxFileSize,
files: 1 // Single file upload by default
},
fileFilter: (req, file, cb) => {
// Basic MIME type check (will be verified again with magic number)
if (allowedMimeTypes.includes(file.mimetype)) {
cb(null, true);
} else {
cb(new Error(`File type ${file.mimetype} not allowed`));
}
}
});
return [
upload.single(fieldName),
createFileSecurityMiddleware({ fileType, allowedMimeTypes })
];
}
module.exports = {
createFileSecurityMiddleware,
createSecureUpload,
validateFileType,
scanWithClamAV,
quarantineFile,
ALLOWED_MIME_TYPES,
MAX_FILE_SIZE
};