tractatus/scripts/sync-instructions-to-db.js

#!/usr/bin/env node

/**
 * Sync Instructions to Database (v3 - Clean programmatic + CLI support)
 */

const fs = require('fs');
const path = require('path');
const mongoose = require('mongoose');

require('dotenv').config();

const GovernanceRule = require('../src/models/GovernanceRule.model');

const INSTRUCTION_FILE = path.join(__dirname, '../.claude/instruction-history.json');
const ORPHAN_BACKUP = path.join(__dirname, '../.claude/backups/orphaned-rules-' + new Date().toISOString().replace(/:/g, '-') + '.json');

// Parse CLI args (only used when run from command line)
const args = process.argv.slice(2);
const cliDryRun = args.includes('--dry-run');
const cliForce = args.includes('--force');
const cliSilent = args.includes('--silent');

const colors = { reset: '\x1b[0m', bright: '\x1b[1m', red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', blue: '\x1b[34m', cyan: '\x1b[36m' };

// Log functions (will respect silent flag passed to main function)
let SILENT = false;
function log(message, color = 'reset') { if (!SILENT) console.log(`${colors[color]}${message}${colors.reset}`); }
function logBright(message) { log(message, 'bright'); }
function logSuccess(message) { log(`✓ ${message}`, 'green'); }
function logWarning(message) { log(`⚠ ${message}`, 'yellow'); }
function logError(message) { log(`✗ ${message}`, 'red'); }
function logInfo(message) { log(`ℹ ${message}`, 'cyan'); }

function mapSource(fileSource) {
  const mapping = {
    'user': 'user_instruction',
    'system': 'framework_default',
    'collaborative': 'user_instruction',
    'framework': 'framework_default',
    'automated': 'automated',
    'migration': 'migration'
  };
  return mapping[fileSource] || 'user_instruction';
}

function mapInstructionToRule(instruction) {
  return {
    id: instruction.id,
    text: instruction.text,
    scope: 'PROJECT_SPECIFIC',
    applicableProjects: ['*'],
    quadrant: instruction.quadrant,
    persistence: instruction.persistence,
    category: mapCategory(instruction),
    priority: mapPriority(instruction),
    temporalScope: instruction.temporal_scope || 'PERMANENT',
    expiresAt: null,
    clarityScore: null,
    specificityScore: null,
    actionabilityScore: null,
    validationStatus: 'NOT_VALIDATED',
    active: instruction.active !== false,
    source: mapSource(instruction.source || 'user'),
    createdBy: 'system',
    createdAt: instruction.timestamp ? new Date(instruction.timestamp) : new Date(),
    notes: instruction.notes || ''
  };
}

function mapCategory(instruction) {
  const text = instruction.text.toLowerCase();
  const quadrant = instruction.quadrant;
  
  if (text.includes('security') || text.includes('csp') || text.includes('auth')) return 'security';
  if (text.includes('privacy') || text.includes('gdpr') || text.includes('consent')) return 'privacy';
  if (text.includes('values') || text.includes('pluralism') || text.includes('legitimacy')) return 'values';
  if (quadrant === 'SYSTEM') return 'technical';
  if (quadrant === 'OPERATIONAL' || quadrant === 'TACTICAL') return 'process';
  return 'other';
}

function mapPriority(instruction) {
  if (instruction.persistence === 'HIGH') return 80;
  if (instruction.persistence === 'MEDIUM') return 50;
  return 30;
}

/**
 * Main sync function
 * @param {Object} options - Sync options
 * @param {boolean} options.silent - Silent mode (default: false)
 * @param {boolean} options.dryRun - Dry run mode (default: false)
 * @param {boolean} options.force - Force sync (default: true when silent)
 */
async function syncInstructions(options = {}) {
  // Determine mode: programmatic call or CLI
  const isDryRun = options.dryRun !== undefined ? options.dryRun : cliDryRun;
  const isSilent = options.silent !== undefined ? options.silent : cliSilent;
  const isForce = options.force !== undefined ? options.force : (cliForce || (!isDryRun && isSilent));

  // Set global silent flag for log functions
  SILENT = isSilent;

  // Track if we created the connection (so we know if we should close it)
  const wasConnected = mongoose.connection.readyState === 1;

  try {
    logBright('\n════════════════════════════════════════════════════════════════');
    logBright('  Tractatus Instruction → Database Sync');
    logBright('════════════════════════════════════════════════════════════════\n');

    if (isDryRun) logInfo('DRY RUN MODE - No changes will be made\n');

    logInfo('Step 1: Reading instruction file...');
    if (!fs.existsSync(INSTRUCTION_FILE)) {
      logError(`Instruction file not found: ${INSTRUCTION_FILE}`);
      return { success: false, error: 'File not found' };
    }

    const fileData = JSON.parse(fs.readFileSync(INSTRUCTION_FILE, 'utf8'));
    const instructions = fileData.instructions || [];

    logSuccess(`Loaded ${instructions.length} instructions from file`);
    log(`   File version: ${fileData.version}`);
    log(`   Last updated: ${fileData.last_updated}\n`);

    logInfo('Step 2: Connecting to MongoDB...');
    const mongoUri = process.env.MONGODB_URI || 'mongodb://localhost:27017/tractatus_dev';
    if (!wasConnected) {
      await mongoose.connect(mongoUri);
      logSuccess(`Connected to MongoDB: ${mongoUri}\n`);
    } else {
      logSuccess(`Using existing MongoDB connection\n`);
    }

    logInfo('Step 3: Analyzing database state...');
    const dbRules = await GovernanceRule.find({}).lean();
    const dbRuleIds = dbRules.map(r => r.id);
    const fileRuleIds = instructions.map(i => i.id);

    log(`   Database has: ${dbRules.length} rules`);
    log(`   File has: ${instructions.length} instructions`);

    const orphanedRules = dbRules.filter(r => !fileRuleIds.includes(r.id));
    const missingRules = instructions.filter(i => !dbRuleIds.includes(i.id));

    log(`   Orphaned (in DB, not in file): ${orphanedRules.length}`);
    log(`   Missing (in file, not in DB): ${missingRules.length}`);
    log(`   Existing (in both): ${instructions.filter(i => dbRuleIds.includes(i.id)).length}\n`);

    if (orphanedRules.length > 0) {
      logWarning('Orphaned rules found:');
      orphanedRules.forEach(r => log(`   - ${r.id}: "${r.text.substring(0, 60)}..."`, 'yellow'));
      log('');
    }

    if (missingRules.length > 0) {
      logInfo('Missing rules (will be added):');
      missingRules.forEach(i => log(`   + ${i.id}: "${i.text.substring(0, 60)}..."`, 'green'));
      log('');
    }

    if (orphanedRules.length > 0 && !isDryRun) {
      logInfo('Step 4: Handling orphaned rules...');
      const backupDir = path.dirname(ORPHAN_BACKUP);
      if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });

      const orphanBackup = {
        timestamp: new Date().toISOString(),
        reason: 'Rules found in MongoDB but not in .claude/instruction-history.json',
        action: 'Soft deleted (marked as inactive)',
        rules: orphanedRules
      };

      fs.writeFileSync(ORPHAN_BACKUP, JSON.stringify(orphanBackup, null, 2));
      logSuccess(`Exported orphaned rules to: ${ORPHAN_BACKUP}`);

      for (const orphan of orphanedRules) {
        await GovernanceRule.findByIdAndUpdate(orphan._id, {
          active: false,
          notes: (orphan.notes || '') + '\n[AUTO-DEACTIVATED: Not found in file-based source of truth on ' + new Date().toISOString() + ']'
        });
      }
      logSuccess(`Deactivated ${orphanedRules.length} orphaned rules\n`);
    } else if (orphanedRules.length > 0 && isDryRun) {
      logInfo('Step 4: [DRY RUN] Would deactivate orphaned rules\n');
    } else {
      logSuccess('Step 4: No orphaned rules found\n');
    }

    logInfo('Step 5: Syncing instructions to database...');
    let addedCount = 0;
    let updatedCount = 0;
    let skippedCount = 0;

    for (const instruction of instructions) {
      const ruleData = mapInstructionToRule(instruction);

      if (isDryRun) {
        if (!dbRuleIds.includes(instruction.id)) {
          log(`   [DRY RUN] Would add: ${instruction.id}`, 'cyan');
          addedCount++;
        } else {
          log(`   [DRY RUN] Would update: ${instruction.id}`, 'cyan');
          updatedCount++;
        }
      } else {
        try {
          const existing = await GovernanceRule.findOne({ id: instruction.id });
          if (existing) {
            await GovernanceRule.findByIdAndUpdate(existing._id, {
              ...ruleData,
              clarityScore: existing.clarityScore || ruleData.clarityScore,
              specificityScore: existing.specificityScore || ruleData.specificityScore,
              actionabilityScore: existing.actionabilityScore || ruleData.actionabilityScore,
              lastOptimized: existing.lastOptimized,
              optimizationHistory: existing.optimizationHistory,
              validationStatus: existing.validationStatus,
              lastValidated: existing.lastValidated,
              validationResults: existing.validationResults,
              updatedAt: new Date()
            });
            updatedCount++;
          } else {
            await GovernanceRule.create(ruleData);
            addedCount++;
          }
        } catch (error) {
          logError(`   Failed to sync ${instruction.id}: ${error.message}`);
          skippedCount++;
        }
      }
    }

    if (isDryRun) {
      log('');
      logInfo('DRY RUN SUMMARY:');
      log(`   Would add: ${addedCount} rules`);
      log(`   Would update: ${updatedCount} rules`);
      log(`   Would skip: ${skippedCount} rules`);
      log(`   Would deactivate: ${orphanedRules.length} orphaned rules\n`);
      logInfo('Run with --force to execute changes\n');
    } else {
      log('');
      logSuccess('SYNC COMPLETE:');
      log(`   Added: ${addedCount} rules`, 'green');
      log(`   Updated: ${updatedCount} rules`, 'green');
      log(`   Skipped: ${skippedCount} rules`, 'yellow');
      log(`   Deactivated: ${orphanedRules.length} orphaned rules`, 'yellow');
      log('');
    }

    logInfo('Step 6: Verifying final state...');
    const finalCount = await GovernanceRule.countDocuments({ active: true });
    const expectedCount = instructions.filter(i => i.active !== false).length;

    if (isDryRun) {
      log(`   Current active rules: ${dbRules.filter(r => r.active).length}`);
      log(`   After sync would be: ${expectedCount}\n`);
    } else {
      log(`   Active rules in database: ${finalCount}`);
      log(`   Expected from file: ${expectedCount}`);
      if (finalCount === expectedCount) {
        logSuccess('   ✓ Counts match!\n');
      } else {
        logWarning(`   ⚠ Mismatch: ${finalCount} vs ${expectedCount}\n`);
      }
    }

    // Only disconnect if we created the connection
    if (!wasConnected && mongoose.connection.readyState === 1) {
      await mongoose.disconnect();
      logSuccess('Disconnected from MongoDB\n');
    } else {
      logSuccess('Leaving connection open for server\n');
    }

    logBright('════════════════════════════════════════════════════════════════');
    if (isDryRun) {
      logInfo('DRY RUN COMPLETE - No changes made');
    } else {
      logSuccess('SYNC COMPLETE');
    }
    logBright('════════════════════════════════════════════════════════════════\n');

    return { success: true, added: addedCount, updated: updatedCount, skipped: skippedCount, deactivated: orphanedRules.length, finalCount: isDryRun ? null : finalCount };

  } catch (error) {
    logError(`\nSync failed: ${error.message}`);
    if (!isSilent) console.error(error.stack);
    // Only disconnect if we created the connection
    if (!wasConnected && mongoose.connection.readyState === 1) {
      await mongoose.disconnect();
    }
    return { success: false, error: error.message };
  }
}

if (require.main === module) {
  if (!cliDryRun && !cliForce && !cliSilent) {
    console.log('\nUsage:');
    console.log('  node scripts/sync-instructions-to-db.js --dry-run  # Preview changes');
    console.log('  node scripts/sync-instructions-to-db.js --force    # Execute sync');
    console.log('  node scripts/sync-instructions-to-db.js --silent   # Background mode\n');
    process.exit(0);
  }

  syncInstructions()
    .then(result => {
      if (result.success) {
        process.exit(0);
      } else {
        process.exit(1);
      }
    })
    .catch(error => {
      console.error('Fatal error:', error);
      process.exit(1);
    });
}

module.exports = { syncInstructions };