tractatus/scripts/cultural-sensitivity-retrospective.js

#!/usr/bin/env node

/**
 * Cultural Sensitivity Retrospective Analysis
 *
 * Phase 3: Learning & Refinement - Retrospective Analysis
 *
 * Runs cultural sensitivity assessment on all existing blog posts
 * to generate baseline data for false positive/negative analysis.
 *
 * Purpose:
 * - Analyze existing content with cultural sensitivity detector
 * - Identify patterns that trigger false positives (appropriate content flagged)
 * - Identify false negatives (insensitive content NOT flagged)
 * - Generate data for refining detection patterns
 *
 * Usage:
 *   node scripts/cultural-sensitivity-retrospective.js [--update-posts]
 *
 * Options:
 *   --update-posts    Update blog posts with cultural sensitivity metadata
 *   --report-only     Generate report without updating database (default)
 */

const mongoose = require('mongoose');
const PluralisticDeliberationOrchestrator = require('../src/services/PluralisticDeliberationOrchestrator.service');
const BlogPost = require('../src/models/BlogPost.model');

const MONGODB_URI = 'mongodb://localhost:27017/tractatus_dev';

// Command line arguments
const UPDATE_POSTS = process.argv.includes('--update-posts');
const REPORT_ONLY = !UPDATE_POSTS;

async function runRetrospectiveAnalysis() {
  console.log('═══════════════════════════════════════════════════════════');
  console.log('  Cultural Sensitivity Retrospective Analysis');
  console.log('  Phase 3: Learning & Refinement');
  console.log('═══════════════════════════════════════════════════════════');
  console.log('');
  console.log(`Mode: ${REPORT_ONLY ? 'REPORT ONLY' : 'UPDATE POSTS'}`);
  console.log('');

  try {
    // Connect to database
    await mongoose.connect(MONGODB_URI);
    console.log('✓ Connected to MongoDB');
    console.log('');

    // Initialize PluralisticDeliberationOrchestrator
    await PluralisticDeliberationOrchestrator.initialize();
    console.log('✓ PluralisticDeliberationOrchestrator initialized');
    console.log('');

    // Fetch all blog posts (all statuses)
    const { getCollection } = require('../src/utils/db.util');
    const collection = await getCollection('blog_posts');
    const posts = await collection.find({}).toArray();
    console.log(`📊 Found ${posts.length} blog posts to analyze`);
    console.log('');

    const results = {
      total: posts.length,
      analyzed: 0,
      low_risk: 0,
      medium_risk: 0,
      high_risk: 0,
      flagged_posts: [],
      all_concerns: [],
      concern_types: {},
      errors: []
    };

    // Analyze each post
    for (const post of posts) {
      console.log(`─────────────────────────────────────────────────────────`);
      console.log(`Analyzing: ${post.title}`);
      console.log(`Slug: ${post.slug}`);
      console.log(`Status: ${post.status}`);
      console.log('');

      try {
        // Get full text for cultural check
        const fullText = [post.title, post.excerpt, post.content]
          .filter(Boolean)
          .join('\n')
          .replace(/<[^>]*>/g, ''); // Strip HTML tags

        // Run cultural sensitivity assessment
        const assessment = await PluralisticDeliberationOrchestrator.assessCulturalSensitivity(fullText, {
          audience: {
            tags: post.tags || []
          },
          content_type: 'blog_post',
          post_id: post._id.toString()
        });

        // Record results
        results.analyzed++;

        if (assessment.risk_level === 'HIGH') results.high_risk++;
        else if (assessment.risk_level === 'MEDIUM') results.medium_risk++;
        else results.low_risk++;

        console.log(`Risk Level: ${assessment.risk_level}`);
        console.log(`Recommended Action: ${assessment.recommended_action}`);
        console.log(`Concerns: ${assessment.concerns.length}`);

        if (assessment.concerns.length > 0) {
          console.log('');
          console.log('Concerns Detected:');
          assessment.concerns.forEach((concern, idx) => {
            console.log(`  ${idx + 1}. [${concern.type}] ${concern.pattern_key}`);
            console.log(`     ${concern.detail}`);
            if (concern.audience_context) {
              console.log(`     Context: ${concern.audience_context}`);
            }
          });

          console.log('');
          console.log('Suggestions:');
          assessment.suggestions.forEach((suggestion, idx) => {
            console.log(`  ${idx + 1}. ${suggestion.suggestion}`);
          });

          results.flagged_posts.push({
            title: post.title,
            slug: post.slug,
            risk_level: assessment.risk_level,
            concerns: assessment.concerns,
            suggestions: assessment.suggestions
          });

          // Track concern types
          assessment.concerns.forEach(concern => {
            const type = concern.pattern_key;
            if (!results.concern_types[type]) {
              results.concern_types[type] = 0;
            }
            results.concern_types[type]++;
            results.all_concerns.push({
              post_title: post.title,
              type: concern.type,
              pattern_key: concern.pattern_key,
              detail: concern.detail
            });
          });
        } else {
          console.log('✓ No cultural sensitivity concerns detected');
        }

        // Update post if requested
        if (UPDATE_POSTS) {
          await collection.updateOne(
            { _id: post._id },
            {
              $set: {
                'moderation.cultural_sensitivity': {
                  risk_level: assessment.risk_level,
                  concerns: assessment.concerns,
                  suggestions: assessment.suggestions,
                  recommended_action: assessment.recommended_action,
                  checked_at: assessment.timestamp
                }
              }
            }
          );
          console.log('✓ Blog post updated with cultural sensitivity metadata');
        }

      } catch (error) {
        console.error(`❌ Error analyzing post: ${error.message}`);
        results.errors.push({
          post_title: post.title,
          error: error.message
        });
      }

      console.log('');
    }

    // Generate report
    console.log('═══════════════════════════════════════════════════════════');
    console.log('  RETROSPECTIVE ANALYSIS REPORT');
    console.log('═══════════════════════════════════════════════════════════');
    console.log('');
    console.log(`Total Posts Analyzed: ${results.analyzed}/${results.total}`);
    console.log(`  LOW risk: ${results.low_risk} (${Math.round(results.low_risk/results.analyzed*100)}%)`);
    console.log(`  MEDIUM risk: ${results.medium_risk} (${Math.round(results.medium_risk/results.analyzed*100)}%)`);
    console.log(`  HIGH risk: ${results.high_risk} (${Math.round(results.high_risk/results.analyzed*100)}%)`);
    console.log('');
    console.log(`Posts Flagged: ${results.flagged_posts.length}/${results.analyzed} (${Math.round(results.flagged_posts.length/results.analyzed*100)}%)`);
    console.log('');

    if (Object.keys(results.concern_types).length > 0) {
      console.log('Concern Types Breakdown:');
      Object.entries(results.concern_types)
        .sort((a, b) => b[1] - a[1])
        .forEach(([type, count]) => {
          console.log(`  ${type}: ${count}`);
        });
      console.log('');
    }

    if (results.flagged_posts.length > 0) {
      console.log('─────────────────────────────────────────────────────────');
      console.log('FLAGGED POSTS FOR REVIEW:');
      console.log('─────────────────────────────────────────────────────────');
      console.log('');
      results.flagged_posts.forEach((post, idx) => {
        console.log(`${idx + 1}. "${post.title}" [${post.risk_level}]`);
        console.log(`   Slug: /blog/${post.slug}`);
        console.log(`   Concerns: ${post.concerns.length}`);
        post.concerns.forEach(concern => {
          console.log(`     • ${concern.pattern_key}: ${concern.detail}`);
        });
        console.log('');
      });
    }

    if (results.errors.length > 0) {
      console.log('─────────────────────────────────────────────────────────');
      console.log('ERRORS:');
      console.log('─────────────────────────────────────────────────────────');
      results.errors.forEach(err => {
        console.log(`  • ${err.post_title}: ${err.error}`);
      });
      console.log('');
    }

    console.log('═══════════════════════════════════════════════════════════');
    console.log('  PHASE 3 ACTION ITEMS');
    console.log('═══════════════════════════════════════════════════════════');
    console.log('');
    console.log('Next Steps:');
    console.log('  1. Review each flagged post for false positives');
    console.log('     (content flagged but culturally appropriate for audience)');
    console.log('  2. Manually review LOW risk posts for false negatives');
    console.log('     (insensitive content that was NOT flagged)');
    console.log('  3. Document patterns in docs/governance/CULTURAL_SENSITIVITY_REFINEMENTS.md');
    console.log('  4. Update detection patterns in PluralisticDeliberationOrchestrator');
    console.log('  5. Re-run this script to validate improvements');
    console.log('');
    console.log('Success Metrics (inst_081):');
    console.log('  • < 10% false positive rate (flagged but appropriate)');
    console.log('  • < 5% false negative rate (missed insensitive content)');
    console.log('');

    if (REPORT_ONLY) {
      console.log('📋 Report generated. No posts updated.');
      console.log('    Run with --update-posts to apply cultural sensitivity metadata to blog posts.');
    } else {
      console.log('✓ Blog posts updated with cultural sensitivity metadata');
    }
    console.log('');

    // Save detailed report to file
    const fs = require('fs');
    const reportPath = '/tmp/cultural-sensitivity-retrospective-' + new Date().toISOString().split('T')[0] + '.json';
    fs.writeFileSync(reportPath, JSON.stringify(results, null, 2));
    console.log(`📄 Detailed report saved: ${reportPath}`);
    console.log('');

    await mongoose.connection.close();
    process.exit(0);

  } catch (err) {
    console.error('');
    console.error('═══════════════════════════════════════════════════════════');
    console.error('ERROR');
    console.error('═══════════════════════════════════════════════════════════');
    console.error(err.message);
    console.error(err.stack);
    await mongoose.connection.close();
    process.exit(1);
  }
}

// Run if called directly
if (require.main === module) {
  runRetrospectiveAnalysis();
}

module.exports = { runRetrospectiveAnalysis };