tractatus/scripts/measure-framework-effectiveness.js

#!/usr/bin/env node

/**
 * Phase 4.2: Framework Effectiveness Metrics
 *
 * Quantitatively measures the impact of Phases 1+2+3:
 * - Framework participation rate
 * - Service utilization distribution
 * - Decision quality indicators
 * - Cross-validation scores (when available)
 */

const mongoose = require('mongoose');

async function main() {
  console.log('═══════════════════════════════════════════════════════════');
  console.log('  FRAMEWORK EFFECTIVENESS METRICS - Phase 4.2');
  console.log('═══════════════════════════════════════════════════════════');
  console.log('');

  // Connect to MongoDB
  try {
    await mongoose.connect('mongodb://localhost:27017/tractatus_dev', {
      serverSelectionTimeoutMS: 5000
    });
    console.log('✓ Connected to MongoDB (tractatus_dev)');
    console.log('');
  } catch (err) {
    console.error('✗ MongoDB connection failed:', err.message);
    process.exit(1);
  }

  // Get audit logs collection
  const db = mongoose.connection.db;
  const auditLogs = db.collection('auditLogs');

  // =============================================================================
  // METRIC 1: Framework Participation Rate
  // =============================================================================

  console.log('METRIC 1: Framework Participation Rate');
  console.log('─────────────────────────────────────────────────────────');

  const totalDecisions = await auditLogs.countDocuments({});
  const frameworkBackedDecisions = await auditLogs.countDocuments({
    'metadata.framework_backed_decision': true
  });

  const participationRate = totalDecisions > 0
    ? ((frameworkBackedDecisions / totalDecisions) * 100).toFixed(1)
    : 0;

  console.log(`  Total Decisions:           ${totalDecisions}`);
  console.log(`  Framework-Backed:          ${frameworkBackedDecisions}`);
  console.log(`  Participation Rate:        ${participationRate}%`);
  console.log('');

  if (participationRate >= 80) {
    console.log('  ✓ EXCELLENT: Framework actively guiding most decisions');
  } else if (participationRate >= 60) {
    console.log('  ✓ GOOD: Framework participating in majority of decisions');
  } else if (participationRate >= 40) {
    console.log('  ⚠ MODERATE: Framework guidance available for some decisions');
  } else {
    console.log('  ✗ LOW: Framework participation needs improvement');
  }
  console.log('');

  // =============================================================================
  // METRIC 2: Service Utilization Distribution
  // =============================================================================

  console.log('METRIC 2: Service Utilization Distribution');
  console.log('─────────────────────────────────────────────────────────');

  const serviceDistribution = await auditLogs.aggregate([
    {
      $group: {
        _id: '$service',
        count: { $sum: 1 },
        frameworkBacked: {
          $sum: {
            $cond: [{ $eq: ['$metadata.framework_backed_decision', true] }, 1, 0]
          }
        }
      }
    },
    { $sort: { count: -1 } }
  ]).toArray();

  console.log('  Service Name                    Total    Framework-Backed    Rate');
  console.log('  ─────────────────────────────   ─────    ────────────────    ────');

  serviceDistribution.forEach(service => {
    const serviceName = service._id || 'Unknown';
    const rate = ((service.frameworkBacked / service.count) * 100).toFixed(0);
    console.log(`  ${serviceName.padEnd(30)} ${String(service.count).padStart(7)}    ${String(service.frameworkBacked).padStart(16)}    ${String(rate).padStart(3)}%`);
  });
  console.log('');

  // =============================================================================
  // METRIC 3: Decision Quality Indicators
  // =============================================================================

  console.log('METRIC 3: Decision Quality Indicators');
  console.log('─────────────────────────────────────────────────────────');

  const allowedDecisions = await auditLogs.countDocuments({ allowed: true });
  const blockedDecisions = await auditLogs.countDocuments({ allowed: false });
  const decisionsWithViolations = await auditLogs.countDocuments({
    violations: { $exists: true, $ne: [] }
  });

  const blockRate = totalDecisions > 0
    ? ((blockedDecisions / totalDecisions) * 100).toFixed(1)
    : 0;

  const violationRate = totalDecisions > 0
    ? ((decisionsWithViolations / totalDecisions) * 100).toFixed(1)
    : 0;

  console.log(`  Allowed Decisions:         ${allowedDecisions} (${((allowedDecisions / totalDecisions) * 100).toFixed(1)}%)`);
  console.log(`  Blocked Decisions:         ${blockedDecisions} (${blockRate}%)`);
  console.log(`  Decisions with Violations: ${decisionsWithViolations} (${violationRate}%)`);
  console.log('');

  if (blockRate < 5) {
    console.log('  ✓ HEALTHY: Low block rate indicates good governance compliance');
  } else if (blockRate < 15) {
    console.log('  ⚠ MODERATE: Some governance violations being caught');
  } else {
    console.log('  ⚠ HIGH: Significant violations being prevented by framework');
  }
  console.log('');

  // =============================================================================
  // METRIC 4: Severity Distribution
  // =============================================================================

  console.log('METRIC 4: Guidance Severity Distribution');
  console.log('─────────────────────────────────────────────────────────');

  const severityDistribution = await auditLogs.aggregate([
    {
      $match: {
        'metadata.guidance_severity': { $exists: true, $ne: null }
      }
    },
    {
      $group: {
        _id: '$metadata.guidance_severity',
        count: { $sum: 1 }
      }
    },
    { $sort: { count: -1 } }
  ]).toArray();

  if (severityDistribution.length > 0) {
    severityDistribution.forEach(severity => {
      const severityName = severity._id || 'Unknown';
      const count = severity.count;
      const pct = ((count / frameworkBackedDecisions) * 100).toFixed(1);
      console.log(`  ${severityName.padEnd(15)} ${String(count).padStart(5)} (${String(pct).padStart(5)}%)`);
    });
  } else {
    console.log('  No severity data available (guidance might not include severity)');
  }
  console.log('');

  // =============================================================================
  // METRIC 5: Temporal Analysis
  // =============================================================================

  console.log('METRIC 5: Temporal Analysis (Last 7 Days)');
  console.log('─────────────────────────────────────────────────────────');

  const sevenDaysAgo = new Date();
  sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);

  const recentDecisions = await auditLogs.countDocuments({
    timestamp: { $gte: sevenDaysAgo }
  });

  const recentFrameworkBacked = await auditLogs.countDocuments({
    timestamp: { $gte: sevenDaysAgo },
    'metadata.framework_backed_decision': true
  });

  const recentParticipationRate = recentDecisions > 0
    ? ((recentFrameworkBacked / recentDecisions) * 100).toFixed(1)
    : 0;

  console.log(`  Recent Decisions (7d):     ${recentDecisions}`);
  console.log(`  Framework-Backed (7d):     ${recentFrameworkBacked}`);
  console.log(`  Recent Participation:      ${recentParticipationRate}%`);
  console.log('');

  if (parseFloat(recentParticipationRate) > parseFloat(participationRate)) {
    console.log(`  ✓ IMPROVING: Recent participation (${recentParticipationRate}%) > Overall (${participationRate}%)`);
  } else if (parseFloat(recentParticipationRate) === parseFloat(participationRate)) {
    console.log(`  → STABLE: Recent participation matches overall rate`);
  } else {
    console.log(`  ⚠ DECLINING: Recent participation (${recentParticipationRate}%) < Overall (${participationRate}%)`);
  }
  console.log('');

  // =============================================================================
  // METRIC 6: Phase 3 Impact Assessment
  // =============================================================================

  console.log('METRIC 6: Phase 3 Impact Assessment');
  console.log('─────────────────────────────────────────────────────────');

  // Check for decisions with guidance objects (Phase 3 feature)
  const decisionsWithGuidance = await auditLogs.countDocuments({
    'metadata.guidance_provided': true
  });

  const guidanceRate = totalDecisions > 0
    ? ((decisionsWithGuidance / totalDecisions) * 100).toFixed(1)
    : 0;

  console.log(`  Decisions with Guidance:   ${decisionsWithGuidance} (${guidanceRate}%)`);
  console.log('');

  if (guidanceRate >= 50) {
    console.log('  ✓ Phase 3 EFFECTIVE: Guidance generation working well');
  } else if (guidanceRate >= 25) {
    console.log('  ⚠ Phase 3 PARTIAL: Some guidance being generated');
  } else {
    console.log('  ✗ Phase 3 LIMITED: Low guidance generation rate');
  }
  console.log('');

  // =============================================================================
  // SUMMARY & RECOMMENDATIONS
  // =============================================================================

  console.log('═══════════════════════════════════════════════════════════');
  console.log('  EFFECTIVENESS SUMMARY');
  console.log('═══════════════════════════════════════════════════════════');
  console.log('');
  console.log('  Key Metrics:');
  console.log(`    • Framework Participation:     ${participationRate}%`);
  console.log(`    • Guidance Generation:         ${guidanceRate}%`);
  console.log(`    • Block Rate:                  ${blockRate}%`);
  console.log(`    • Violation Detection:         ${violationRate}%`);
  console.log(`    • Active Services:             ${serviceDistribution.length}`);
  console.log('');

  // Calculate overall effectiveness score
  const effectivenessScore = Math.round(
    (parseFloat(participationRate) * 0.4) +
    (parseFloat(guidanceRate) * 0.3) +
    ((100 - parseFloat(violationRate)) * 0.2) +
    (Math.min(serviceDistribution.length / 6, 1) * 100 * 0.1)
  );

  console.log(`  Overall Effectiveness Score: ${effectivenessScore}/100`);
  console.log('');

  if (effectivenessScore >= 80) {
    console.log('  🎉 EXCELLENT: Framework is highly effective');
  } else if (effectivenessScore >= 60) {
    console.log('  ✓ GOOD: Framework is performing well');
  } else if (effectivenessScore >= 40) {
    console.log('  ⚠ MODERATE: Framework needs improvement');
  } else {
    console.log('  ✗ LOW: Framework requires significant tuning');
  }
  console.log('');

  console.log('  Recommendations for Phase 4.3:');
  if (parseFloat(participationRate) < 70) {
    console.log('    • Increase framework participation rate (currently ' + participationRate + '%)');
  }
  if (parseFloat(guidanceRate) < 60) {
    console.log('    • Improve guidance generation coverage (currently ' + guidanceRate + '%)');
  }
  if (parseFloat(blockRate) > 10) {
    console.log('    • Review keyword lists - block rate seems high (' + blockRate + '%)');
  }
  if (serviceDistribution.length < 5) {
    console.log('    • Ensure all framework services are being utilized');
  }
  console.log('');

  console.log('═══════════════════════════════════════════════════════════');

  await mongoose.disconnect();
  process.exit(0);
}

main().catch(err => {
  console.error('Fatal error:', err);
  process.exit(1);
});