tractatus/scripts/analyze-instruction-violations.js

#!/usr/bin/env node

/**
 * Instruction History Analytics
 *
 * Analyzes instruction-history.json to provide insights into:
 * - Instruction usage patterns
 * - Most violated instructions
 * - Instructions never referenced
 * - Quadrant distribution
 * - Enforcement effectiveness
 *
 * Usage: node scripts/analyze-instruction-violations.js
 */

const fs = require('fs');
const path = require('path');

const INSTRUCTION_HISTORY_PATH = path.join(__dirname, '../.claude/instruction-history.json');
const INCIDENTS_PATH = path.join(__dirname, '../.claude/framework-incidents.json');

/**
 * Color output helpers
 */
const colors = {
  reset: '\x1b[0m',
  bright: '\x1b[1m',
  green: '\x1b[32m',
  yellow: '\x1b[33m',
  blue: '\x1b[34m',
  red: '\x1b[31m',
  cyan: '\x1b[36m',
  magenta: '\x1b[35m'
};

function log(message, color = 'reset') {
  console.log(`${colors[color]}${message}${colors.reset}`);
}

function header(message) {
  console.log('');
  log('═'.repeat(80), 'cyan');
  log(`  ${message}`, 'bright');
  log('═'.repeat(80), 'cyan');
  console.log('');
}

function section(message) {
  console.log('');
  log(`▶ ${message}`, 'blue');
}

/**
 * Load instruction history
 */
function loadInstructions() {
  try {
    const data = fs.readFileSync(INSTRUCTION_HISTORY_PATH, 'utf8');
    return JSON.parse(data);
  } catch (err) {
    log(`Error loading instruction history: ${err.message}`, 'red');
    process.exit(1);
  }
}

/**
 * Load framework incidents (if exists)
 */
function loadIncidents() {
  try {
    if (!fs.existsSync(INCIDENTS_PATH)) {
      return { incidents: [] };
    }
    const data = fs.readFileSync(INCIDENTS_PATH, 'utf8');
    return JSON.parse(data);
  } catch (err) {
    log(`Warning: Could not load incidents database: ${err.message}`, 'yellow');
    return { incidents: [] };
  }
}

/**
 * Analyze instruction usage
 */
function analyzeInstructions(history) {
  const instructions = history.instructions || [];
  const active = instructions.filter(i => i.active);
  const inactive = instructions.filter(i => !i.active);

  // Quadrant distribution
  const byQuadrant = {
    STRATEGIC: active.filter(i => i.quadrant === 'STRATEGIC').length,
    OPERATIONAL: active.filter(i => i.quadrant === 'OPERATIONAL').length,
    TACTICAL: active.filter(i => i.quadrant === 'TACTICAL').length,
    SYSTEM: active.filter(i => i.quadrant === 'SYSTEM').length,
    STOCHASTIC: active.filter(i => i.quadrant === 'STOCHASTIC').length
  };

  // Persistence distribution
  const byPersistence = {
    HIGH: active.filter(i => i.persistence === 'HIGH').length,
    MEDIUM: active.filter(i => i.persistence === 'MEDIUM').length,
    LOW: active.filter(i => i.persistence === 'LOW').length,
    VARIABLE: active.filter(i => i.persistence === 'VARIABLE').length
  };

  // Temporal scope distribution
  const byScope = {
    PERMANENT: active.filter(i => i.temporal_scope === 'PERMANENT').length,
    PROJECT: active.filter(i => i.temporal_scope === 'PROJECT').length,
    PHASE: active.filter(i => i.temporal_scope === 'PHASE').length,
    SESSION: active.filter(i => i.temporal_scope === 'SESSION').length
  };

  return {
    total: instructions.length,
    active: active.length,
    inactive: inactive.length,
    byQuadrant,
    byPersistence,
    byScope,
    instructions: active
  };
}

/**
 * Analyze violations from incidents database
 */
function analyzeViolations(incidents) {
  const violations = {};

  incidents.forEach(incident => {
    const instId = incident.instruction_violated || incident.id;
    if (!violations[instId]) {
      violations[instId] = {
        count: 0,
        tokens_wasted: 0,
        incidents: []
      };
    }
    violations[instId].count++;
    violations[instId].tokens_wasted += incident.tokens_wasted || 0;
    violations[instId].incidents.push(incident);
  });

  return violations;
}

/**
 * Find never-referenced instructions
 */
function findUnusedInstructions(instructions, violations) {
  const neverViolated = instructions.filter(i => !violations[i.id]);

  return {
    neverViolated: neverViolated.map(i => i.id),
    strategicNeverViolated: neverViolated.filter(i => i.quadrant === 'STRATEGIC').map(i => i.id)
  };
}

/**
 * Calculate enforcement effectiveness
 */
function calculateEnforcement(instructions, violations) {
  // Hook-enforced instructions (those with architectural enforcement)
  const hookEnforced = instructions.filter(i =>
    i.enforcement === 'architectural' ||
    i.enforcement === 'hook' ||
    i.notes?.includes('hook') ||
    i.notes?.includes('architectural')
  );

  const voluntary = instructions.filter(i =>
    !hookEnforced.includes(i)
  );

  // Calculate violation rates
  const hookEnforcedViolations = hookEnforced.filter(i => violations[i.id]);
  const voluntaryViolations = voluntary.filter(i => violations[i.id]);

  const hookEnforcementRate = hookEnforced.length > 0
    ? ((hookEnforced.length - hookEnforcedViolations.length) / hookEnforced.length * 100).toFixed(1)
    : 0;

  const voluntaryComplianceRate = voluntary.length > 0
    ? ((voluntary.length - voluntaryViolations.length) / voluntary.length * 100).toFixed(1)
    : 0;

  return {
    hookEnforced: hookEnforced.length,
    voluntary: voluntary.length,
    hookEnforcementRate,
    voluntaryComplianceRate,
    hookEnforcedViolations: hookEnforcedViolations.length,
    voluntaryViolations: voluntaryViolations.length
  };
}

/**
 * Main analytics
 */
function main() {
  header('Instruction History Analytics');

  // Load data
  const history = loadInstructions();
  const incidentsData = loadIncidents();
  const incidents = incidentsData.incidents || [];

  log(`📊 Analyzing ${history.instructions?.length || 0} instructions and ${incidents.length} incidents`, 'cyan');

  // Analyze
  const analysis = analyzeInstructions(history);
  const violations = analyzeViolations(incidents);
  const unused = findUnusedInstructions(analysis.instructions, violations);
  const enforcement = calculateEnforcement(analysis.instructions, violations);

  // Display results

  // 1. Overview
  section('1. Instruction Overview');
  log(`  Total instructions: ${analysis.total}`, 'cyan');
  log(`  Active: ${analysis.active}`, 'green');
  log(`  Inactive: ${analysis.inactive}`, 'yellow');

  // 2. Distribution by Quadrant
  section('2. Distribution by Quadrant');
  Object.entries(analysis.byQuadrant).forEach(([quadrant, count]) => {
    const bar = '█'.repeat(Math.ceil(count / 2));
    log(`  ${quadrant.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan');
  });

  // 3. Distribution by Persistence
  section('3. Distribution by Persistence');
  Object.entries(analysis.byPersistence).forEach(([level, count]) => {
    const bar = '█'.repeat(Math.ceil(count / 2));
    log(`  ${level.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan');
  });

  // 4. Distribution by Temporal Scope
  section('4. Distribution by Temporal Scope');
  Object.entries(analysis.byScope).forEach(([scope, count]) => {
    const bar = '█'.repeat(Math.ceil(count / 2));
    log(`  ${scope.padEnd(12)}: ${count.toString().padStart(2)} ${bar}`, 'cyan');
  });

  // 5. Most Violated Instructions
  section('5. Most Violated Instructions');
  const violationList = Object.entries(violations)
    .sort((a, b) => b[1].count - a[1].count);

  if (violationList.length === 0) {
    log('  ✅ No violations recorded', 'green');
  } else {
    violationList.slice(0, 10).forEach(([instId, data]) => {
      const instruction = analysis.instructions.find(i => i.id === instId);
      const text = instruction ? instruction.text.substring(0, 60) + '...' : 'Unknown instruction';
      log(`  ${instId}: ${data.count} violation(s), ${data.tokens_wasted.toLocaleString()} tokens wasted`, 'red');
      log(`    "${text}"`, 'yellow');
    });
  }

  // 6. Never Violated Instructions
  section('6. Never Violated Instructions');
  if (unused.neverViolated.length === 0) {
    log('  ⚠️  All instructions have been violated at least once!', 'yellow');
  } else {
    log(`  ${unused.neverViolated.length} instructions with 100% compliance:`, 'green');
    unused.neverViolated.slice(0, 10).forEach(instId => {
      const instruction = analysis.instructions.find(i => i.id === instId);
      if (instruction) {
        log(`    ${instId}: ${instruction.text.substring(0, 70)}`, 'cyan');
      }
    });
    if (unused.neverViolated.length > 10) {
      log(`    ... and ${unused.neverViolated.length - 10} more`, 'cyan');
    }
  }

  // 7. Enforcement Effectiveness
  section('7. Enforcement Effectiveness');
  log(`  Hook-enforced instructions: ${enforcement.hookEnforced}`, 'cyan');
  log(`    Violations: ${enforcement.hookEnforcedViolations}`, enforcement.hookEnforcedViolations > 0 ? 'red' : 'green');
  log(`    Compliance rate: ${enforcement.hookEnforcementRate}%`, enforcement.hookEnforcementRate >= 95 ? 'green' : 'yellow');

  console.log('');
  log(`  Voluntary compliance instructions: ${enforcement.voluntary}`, 'cyan');
  log(`    Violations: ${enforcement.voluntaryViolations}`, enforcement.voluntaryViolations > 0 ? 'red' : 'green');
  log(`    Compliance rate: ${enforcement.voluntaryComplianceRate}%`, enforcement.voluntaryComplianceRate >= 95 ? 'green' : 'yellow');

  console.log('');
  if (enforcement.hookEnforcementRate > enforcement.voluntaryComplianceRate) {
    log(`  ✅ Hook enforcement is ${(enforcement.hookEnforcementRate - enforcement.voluntaryComplianceRate).toFixed(1)}% more effective`, 'green');
    log(`  💡 Recommendation: Convert more voluntary compliance to architectural enforcement`, 'cyan');
  } else if (enforcement.voluntaryComplianceRate >= 95) {
    log(`  ✅ Voluntary compliance is working well (${enforcement.voluntaryComplianceRate}%)`, 'green');
  } else {
    log(`  ⚠️  Consider improving enforcement mechanisms`, 'yellow');
  }

  // 8. Recommendations
  section('8. Recommendations');

  const recommendations = [];

  // High-violation instructions needing enforcement
  const highViolation = violationList.filter(([_, data]) => data.count >= 2);
  if (highViolation.length > 0) {
    recommendations.push({
      priority: 'HIGH',
      text: `${highViolation.length} instruction(s) violated 2+ times - add architectural enforcement`,
      details: highViolation.map(([id]) => id)
    });
  }

  // Strategic instructions that are never violated (good!)
  if (unused.strategicNeverViolated.length > 0) {
    recommendations.push({
      priority: 'INFO',
      text: `${unused.strategicNeverViolated.length} STRATEGIC instructions have 100% compliance - document success`,
      details: unused.strategicNeverViolated.slice(0, 5)
    });
  }

  // Voluntary compliance gaps
  if (enforcement.voluntaryComplianceRate < 80) {
    recommendations.push({
      priority: 'MEDIUM',
      text: `Voluntary compliance at ${enforcement.voluntaryComplianceRate}% - convert to hooks or improve documentation`,
      details: []
    });
  }

  if (recommendations.length === 0) {
    log('  ✅ No recommendations - framework is performing well!', 'green');
  } else {
    recommendations.forEach((rec, i) => {
      const color = rec.priority === 'HIGH' ? 'red' : rec.priority === 'MEDIUM' ? 'yellow' : 'cyan';
      log(`  ${i + 1}. [${rec.priority}] ${rec.text}`, color);
      if (rec.details.length > 0) {
        rec.details.forEach(detail => {
          log(`     - ${detail}`, 'cyan');
        });
      }
    });
  }

  // Summary footer
  header('Analytics Complete');
  console.log('');
  log(`  📈 Key Metrics:`, 'bright');
  log(`     Active Instructions: ${analysis.active}`, 'cyan');
  log(`     Recorded Violations: ${incidents.length}`, incidents.length > 0 ? 'yellow' : 'green');
  log(`     Tokens Wasted: ${Object.values(violations).reduce((sum, v) => sum + v.tokens_wasted, 0).toLocaleString()}`, 'red');
  log(`     Hook Enforcement Rate: ${enforcement.hookEnforcementRate}%`, 'green');
  log(`     Voluntary Compliance Rate: ${enforcement.voluntaryComplianceRate}%`, 'yellow');
  console.log('');
  log(`  💡 Next Steps:`, 'bright');
  log(`     - Review high-violation instructions for enforcement gaps`, 'cyan');
  log(`     - Document successful compliance patterns`, 'cyan');
  log(`     - Convert voluntary → architectural where violations occur`, 'cyan');
  console.log('');
}

// Run
main();