#!/usr/bin/env node /** * Instruction History Analytics * * Analyzes instruction-history.json to provide insights into: * - Instruction usage patterns * - Most violated instructions * - Instructions never referenced * - Quadrant distribution * - Enforcement effectiveness * * Usage: node scripts/analyze-instruction-violations.js */ const fs = require('fs'); const path = require('path'); const INSTRUCTION_HISTORY_PATH = path.join(__dirname, '../.claude/instruction-history.json'); const INCIDENTS_PATH = path.join(__dirname, '../.claude/framework-incidents.json'); /** * Color output helpers */ const colors = { reset: '\x1b[0m', bright: '\x1b[1m', green: '\x1b[32m', yellow: '\x1b[33m', blue: '\x1b[34m', red: '\x1b[31m', cyan: '\x1b[36m', magenta: '\x1b[35m' }; function log(message, color = 'reset') { console.log(`${colors[color]}${message}${colors.reset}`); } function header(message) { console.log(''); log('═'.repeat(80), 'cyan'); log(` ${message}`, 'bright'); log('═'.repeat(80), 'cyan'); console.log(''); } function section(message) { console.log(''); log(`▶ ${message}`, 'blue'); } /** * Load instruction history */ function loadInstructions() { try { const data = fs.readFileSync(INSTRUCTION_HISTORY_PATH, 'utf8'); return JSON.parse(data); } catch (err) { log(`Error loading instruction history: ${err.message}`, 'red'); process.exit(1); } } /** * Load framework incidents (if exists) */ function loadIncidents() { try { if (!fs.existsSync(INCIDENTS_PATH)) { return { incidents: [] }; } const data = fs.readFileSync(INCIDENTS_PATH, 'utf8'); return JSON.parse(data); } catch (err) { log(`Warning: Could not load incidents database: ${err.message}`, 'yellow'); return { incidents: [] }; } } /** * Analyze instruction usage */ function analyzeInstructions(history) { const instructions = history.instructions || []; const active = instructions.filter(i => i.active); const inactive = instructions.filter(i => !i.active); // Quadrant distribution const byQuadrant = { STRATEGIC: active.filter(i => i.quadrant === 'STRATEGIC').length, OPERATIONAL: active.filter(i => i.quadrant === 'OPERATIONAL').length, TACTICAL: active.filter(i => i.quadrant === 'TACTICAL').length, SYSTEM: active.filter(i => i.quadrant === 'SYSTEM').length, STOCHASTIC: active.filter(i => i.quadrant === 'STOCHASTIC').length }; // Persistence distribution const byPersistence = { HIGH: active.filter(i => i.persistence === 'HIGH').length, MEDIUM: active.filter(i => i.persistence === 'MEDIUM').length, LOW: active.filter(i => i.persistence === 'LOW').length, VARIABLE: active.filter(i => i.persistence === 'VARIABLE').length }; // Temporal scope distribution const byScope = { PERMANENT: active.filter(i => i.temporal_scope === 'PERMANENT').length, PROJECT: active.filter(i => i.temporal_scope === 'PROJECT').length, PHASE: active.filter(i => i.temporal_scope === 'PHASE').length, SESSION: active.filter(i => i.temporal_scope === 'SESSION').length }; return { total: instructions.length, active: active.length, inactive: inactive.length, byQuadrant, byPersistence, byScope, instructions: active }; } /** * Analyze violations from incidents database */ function analyzeViolations(incidents) { const violations = {}; incidents.forEach(incident => { const instId = incident.instruction_violated || incident.id; if (!violations[instId]) { violations[instId] = { count: 0, tokens_wasted: 0, incidents: [] }; } violations[instId].count++; violations[instId].tokens_wasted += incident.tokens_wasted || 0; violations[instId].incidents.push(incident); }); return violations; } /** * Find never-referenced instructions */ function findUnusedInstructions(instructions, violations) { const neverViolated = instructions.filter(i => !violations[i.id]); return { neverViolated: neverViolated.map(i => i.id), strategicNeverViolated: neverViolated.filter(i => i.quadrant === 'STRATEGIC').map(i => i.id) }; } /** * Calculate enforcement effectiveness */ function calculateEnforcement(instructions, violations) { // Hook-enforced instructions (those with architectural enforcement) const hookEnforced = instructions.filter(i => i.enforcement === 'architectural' || i.enforcement === 'hook' || i.notes?.includes('hook') || i.notes?.includes('architectural') ); const voluntary = instructions.filter(i => !hookEnforced.includes(i) ); // Calculate violation rates const hookEnforcedViolations = hookEnforced.filter(i => violations[i.id]); const voluntaryViolations = voluntary.filter(i => violations[i.id]); const hookEnforcementRate = hookEnforced.length > 0 ? ((hookEnforced.length - hookEnforcedViolations.length) / hookEnforced.length * 100).toFixed(1) : 0; const voluntaryComplianceRate = voluntary.length > 0 ? ((voluntary.length - voluntaryViolations.length) / voluntary.length * 100).toFixed(1) : 0; return { hookEnforced: hookEnforced.length, voluntary: voluntary.length, hookEnforcementRate, voluntaryComplianceRate, hookEnforcedViolations: hookEnforcedViolations.length, voluntaryViolations: voluntaryViolations.length }; } /** * Main analytics */ function main() { header('Instruction History Analytics'); // Load data const history = loadInstructions(); const incidentsData = loadIncidents(); const incidents = incidentsData.incidents || []; log(`📊 Analyzing ${history.instructions?.length || 0} instructions and ${incidents.length} incidents`, 'cyan'); // Analyze const analysis = analyzeInstructions(history); const violations = analyzeViolations(incidents); const unused = findUnusedInstructions(analysis.instructions, violations); const enforcement = calculateEnforcement(analysis.instructions, violations); // Display results // 1. Overview section('1. Instruction Overview'); log(` Total instructions: ${analysis.total}`, 'cyan'); log(` Active: ${analysis.active}`, 'green'); log(` Inactive: ${analysis.inactive}`, 'yellow'); // 2. Distribution by Quadrant section('2. Distribution by Quadrant'); Object.entries(analysis.byQuadrant).forEach(([quadrant, count]) => { const bar = '█'.repeat(Math.ceil(count / 2)); log(` ${quadrant.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan'); }); // 3. Distribution by Persistence section('3. Distribution by Persistence'); Object.entries(analysis.byPersistence).forEach(([level, count]) => { const bar = '█'.repeat(Math.ceil(count / 2)); log(` ${level.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan'); }); // 4. Distribution by Temporal Scope section('4. Distribution by Temporal Scope'); Object.entries(analysis.byScope).forEach(([scope, count]) => { const bar = '█'.repeat(Math.ceil(count / 2)); log(` ${scope.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan'); }); // 5. Most Violated Instructions section('5. Most Violated Instructions'); const violationList = Object.entries(violations) .sort((a, b) => b[1].count - a[1].count); if (violationList.length === 0) { log(' ✅ No violations recorded', 'green'); } else { violationList.slice(0, 10).forEach(([instId, data]) => { const instruction = analysis.instructions.find(i => i.id === instId); const text = instruction ? instruction.text.substring(0, 60) + '...' : 'Unknown instruction'; log(` ${instId}: ${data.count} violation(s), ${data.tokens_wasted.toLocaleString()} tokens wasted`, 'red'); log(` "${text}"`, 'yellow'); }); } // 6. Never Violated Instructions section('6. Never Violated Instructions'); if (unused.neverViolated.length === 0) { log(' ⚠️ All instructions have been violated at least once!', 'yellow'); } else { log(` ${unused.neverViolated.length} instructions with 100% compliance:`, 'green'); unused.neverViolated.slice(0, 10).forEach(instId => { const instruction = analysis.instructions.find(i => i.id === instId); if (instruction) { log(` ${instId}: ${instruction.text.substring(0, 70)}`, 'cyan'); } }); if (unused.neverViolated.length > 10) { log(` ... and ${unused.neverViolated.length - 10} more`, 'cyan'); } } // 7. Enforcement Effectiveness section('7. Enforcement Effectiveness'); log(` Hook-enforced instructions: ${enforcement.hookEnforced}`, 'cyan'); log(` Violations: ${enforcement.hookEnforcedViolations}`, enforcement.hookEnforcedViolations > 0 ? 'red' : 'green'); log(` Compliance rate: ${enforcement.hookEnforcementRate}%`, enforcement.hookEnforcementRate >= 95 ? 'green' : 'yellow'); console.log(''); log(` Voluntary compliance instructions: ${enforcement.voluntary}`, 'cyan'); log(` Violations: ${enforcement.voluntaryViolations}`, enforcement.voluntaryViolations > 0 ? 'red' : 'green'); log(` Compliance rate: ${enforcement.voluntaryComplianceRate}%`, enforcement.voluntaryComplianceRate >= 95 ? 'green' : 'yellow'); console.log(''); if (enforcement.hookEnforcementRate > enforcement.voluntaryComplianceRate) { log(` ✅ Hook enforcement is ${(enforcement.hookEnforcementRate - enforcement.voluntaryComplianceRate).toFixed(1)}% more effective`, 'green'); log(` 💡 Recommendation: Convert more voluntary compliance to architectural enforcement`, 'cyan'); } else if (enforcement.voluntaryComplianceRate >= 95) { log(` ✅ Voluntary compliance is working well (${enforcement.voluntaryComplianceRate}%)`, 'green'); } else { log(` ⚠️ Consider improving enforcement mechanisms`, 'yellow'); } // 8. Recommendations section('8. Recommendations'); const recommendations = []; // High-violation instructions needing enforcement const highViolation = violationList.filter(([_, data]) => data.count >= 2); if (highViolation.length > 0) { recommendations.push({ priority: 'HIGH', text: `${highViolation.length} instruction(s) violated 2+ times - add architectural enforcement`, details: highViolation.map(([id]) => id) }); } // Strategic instructions that are never violated (good!) if (unused.strategicNeverViolated.length > 0) { recommendations.push({ priority: 'INFO', text: `${unused.strategicNeverViolated.length} STRATEGIC instructions have 100% compliance - document success`, details: unused.strategicNeverViolated.slice(0, 5) }); } // Voluntary compliance gaps if (enforcement.voluntaryComplianceRate < 80) { recommendations.push({ priority: 'MEDIUM', text: `Voluntary compliance at ${enforcement.voluntaryComplianceRate}% - convert to hooks or improve documentation`, details: [] }); } if (recommendations.length === 0) { log(' ✅ No recommendations - framework is performing well!', 'green'); } else { recommendations.forEach((rec, i) => { const color = rec.priority === 'HIGH' ? 'red' : rec.priority === 'MEDIUM' ? 'yellow' : 'cyan'; log(` ${i + 1}. [${rec.priority}] ${rec.text}`, color); if (rec.details.length > 0) { rec.details.forEach(detail => { log(` - ${detail}`, 'cyan'); }); } }); } // Summary footer header('Analytics Complete'); console.log(''); log(` 📈 Key Metrics:`, 'bright'); log(` Active Instructions: ${analysis.active}`, 'cyan'); log(` Recorded Violations: ${incidents.length}`, incidents.length > 0 ? 'yellow' : 'green'); log(` Tokens Wasted: ${Object.values(violations).reduce((sum, v) => sum + v.tokens_wasted, 0).toLocaleString()}`, 'red'); log(` Hook Enforcement Rate: ${enforcement.hookEnforcementRate}%`, 'green'); log(` Voluntary Compliance Rate: ${enforcement.voluntaryComplianceRate}%`, 'yellow'); console.log(''); log(` 💡 Next Steps:`, 'bright'); log(` - Review high-violation instructions for enforcement gaps`, 'cyan'); log(` - Document successful compliance patterns`, 'cyan'); log(` - Convert voluntary → architectural where violations occur`, 'cyan'); console.log(''); } // Run main();