tractatus/scripts/analyze-degradation.js

/**
 * Analyze Performance Degradation Patterns
 * Identifies missing metrics in pressure gauge
 */

const { MongoClient } = require('mongodb');

async function analyzeDegradation() {
  const client = new MongoClient('mongodb://localhost:27017');
  await client.connect();
  const db = client.db('tractatus_dev');

  const today = new Date('2025-11-04T00:00:00Z');

  console.log('\n=== SESSION DEGRADATION ANALYSIS ===\n');

  // 1. Error accumulation pattern (today)
  const errors = await db.collection('framework_audit').find({
    timestamp: { $gte: today },
    $or: [
      { 'decision.blocked': true },
      { 'decision.errors': { $exists: true } }
    ]
  }).toArray();

  console.log(`Total Errors Today: ${errors.length}`);

  // Group by hour
  const errorsByHour = {};
  errors.forEach(e => {
    const hour = new Date(e.timestamp).getHours();
    errorsByHour[hour] = (errorsByHour[hour] || 0) + 1;
  });

  console.log('\nErrors by Hour:');
  Object.entries(errorsByHour).forEach(([hour, count]) => {
    console.log(`  ${hour}:00 - ${count} errors`);
  });

  // 2. Consecutive error detection
  const recentAudits = await db.collection('framework_audit')
    .find({ timestamp: { $gte: today } })
    .sort({ timestamp: 1 })
    .limit(100)
    .toArray();

  let consecutiveErrors = 0;
  let maxConsecutive = 0;
  let errorClusters = [];
  let currentCluster = [];

  recentAudits.forEach((a, i) => {
    const hasError = a.decision?.blocked || a.decision?.errors;
    if (hasError) {
      consecutiveErrors++;
      currentCluster.push({ service: a.service, time: a.timestamp });
      maxConsecutive = Math.max(maxConsecutive, consecutiveErrors);
    } else {
      if (currentCluster.length > 2) {
        errorClusters.push([...currentCluster]);
      }
      currentCluster = [];
      consecutiveErrors = 0;
    }
  });

  console.log(`\nConsecutive Error Patterns:`);
  console.log(`  Max consecutive errors: ${maxConsecutive}`);
  console.log(`  Error clusters (3+ errors): ${errorClusters.length}`);

  // 3. Framework component activity
  const componentActivity = await db.collection('framework_audit').aggregate([
    { $match: { timestamp: { $gte: today } } },
    {
      $group: {
        _id: '$service',
        count: { $sum: 1 },
        lastActivity: { $max: '$timestamp' }
      }
    }
  ]).toArray();

  console.log('\nFramework Component Activity:');
  const allComponents = [
    'ContextPressureMonitor',
    'BoundaryEnforcer',
    'MetacognitiveVerifier',
    'CrossReferenceValidator',
    'InstructionPersistenceClassifier',
    'PluralisticDeliberationOrchestrator'
  ];

  allComponents.forEach(comp => {
    const activity = componentActivity.find(c => c._id === comp);
    if (activity) {
      const age = (new Date() - activity.lastActivity) / 1000 / 60;
      console.log(`  ✓ ${comp}: ${activity.count} calls, ${age.toFixed(0)}m ago`);
    } else {
      console.log(`  ✗ ${comp}: NEVER USED (framework fade!)`);
    }
  });

  // 4. Session state analysis
  const session = await db.collection('session_state').findOne({ sessionId: '2025-10-07-001' });

  console.log('\n=== CURRENT PRESSURE GAUGE ===');
  console.log(`Metrics tracked:`);
  console.log(`  • tokenUsage: ${session?.contextPressure?.metrics?.tokenUsage?.percentage || 'N/A'}%`);
  console.log(`  • conversationLength: ${session?.contextPressure?.metrics?.conversationLength?.value || 0}`);
  console.log(`  • taskComplexity: ${session?.contextPressure?.metrics?.taskComplexity?.value || 0}`);
  console.log(`  • errorFrequency: ${session?.contextPressure?.metrics?.errorFrequency?.raw || 0} (simple count)`);
  console.log(`  • instructionDensity: ${session?.contextPressure?.metrics?.instructionDensity?.value || 0}`);
  console.log(`\nOverall Score: ${session?.contextPressure?.score || 0}%`);

  console.log('\n=== MISSING DEGRADATION METRICS ===');
  console.log('\n1. ERROR PATTERN ANALYSIS:');
  console.log('   ✗ Consecutive error count (current: ' + maxConsecutive + ')');
  console.log('   ✗ Error clustering detection');
  console.log('   ✗ Repeated failures on similar tasks');
  console.log('   ✗ Error severity weighting');

  console.log('\n2. FRAMEWORK HEALTH:');
  console.log('   ✗ Framework component fade detection');
  console.log('   ✗ MetacognitiveVerifier usage frequency');
  console.log('   ✗ Time since last successful task completion');

  console.log('\n3. CONTEXT QUALITY:');
  console.log('   ✗ Post-compaction context degradation');
  console.log('   ✗ Knowledge domain shift detection');
  console.log('   ✗ Continued session after compaction flag');

  console.log('\n4. BEHAVIORAL INDICATORS:');
  console.log('   ✗ Tool retry rate (same tool called multiple times)');
  console.log('   ✗ File read without subsequent action rate');
  console.log('   ✗ Deployment/restart frequency (thrashing)');

  console.log('\n=== RECOMMENDATION ===');
  console.log('\nAdd new pressure metric: "degradationScore" that combines:');
  console.log('  1. Consecutive errors (weight: 0.3)');
  console.log('  2. Framework fade (weight: 0.25)');
  console.log('  3. Error clustering (weight: 0.2)');
  console.log('  4. Post-compaction flag (weight: 0.15)');
  console.log('  5. Tool retry rate (weight: 0.1)');
  console.log('\nThreshold: degradationScore > 40% = WARN user');
  console.log('Threshold: degradationScore > 60% = RECOMMEND session restart');

  await client.close();
}

analyzeDegradation().catch(console.error);