tractatus/scripts/migrate-value-pluralism-docs.js

/**
 * Migrate Value Pluralism Documents to MongoDB
 * Adds three new value pluralism documents to the documents collection
 */

// Load environment variables from .env file
require('dotenv').config();

const mongoose = require('mongoose');
const marked = require('marked');
const fs = require('fs').promises;
const path = require('path');
const config = require('../src/config/app.config');

// Document structure from existing documents
const documentsToMigrate = [
  {
    file: 'docs/research/pluralistic-values-research-foundations.md',
    title: 'Pluralistic Values: Research Foundations',
    slug: 'pluralistic-values-research-foundations',
    quadrant: 'STRATEGIC',
    persistence: 'HIGH',
    category: 'research',
    audience: ['researcher', 'technical'],
    tags: ['value-pluralism', 'research', 'deliberative-democracy', 'ethics', 'philosophy'],
    description: 'Supporting research material for PluralisticDeliberationOrchestrator implementation, covering deliberative democracy, value pluralism theory, and cross-cultural communication'
  },
  {
    file: 'docs/value-pluralism-faq.md',
    title: 'Value Pluralism in Tractatus: Frequently Asked Questions',
    slug: 'value-pluralism-faq',
    quadrant: null,
    persistence: 'HIGH',
    category: 'documentation',
    audience: ['general'],
    tags: ['value-pluralism', 'faq', 'documentation', 'ethics'],
    description: 'Accessible explanation of how Tractatus handles moral disagreement without imposing hierarchy'
  },
  {
    file: 'docs/pluralistic-values-deliberation-plan-v2.md',
    title: 'Pluralistic Values Deliberation Enhancement Plan',
    slug: 'pluralistic-values-deliberation-plan-v2',
    quadrant: 'OPERATIONAL',
    persistence: 'HIGH',
    category: 'implementation-guide',
    audience: ['implementer', 'researcher'],
    tags: ['value-pluralism', 'implementation', 'deliberation', 'planning'],
    description: 'Technical design document for implementing non-hierarchical moral reasoning in the Tractatus Framework'
  }
];

/**
 * Generate table of contents from markdown
 */
function generateToC(markdown) {
  const toc = [];
  const lines = markdown.split('\n');

  for (const line of lines) {
    const match = line.match(/^(#{1,6})\s+(.+)$/);
    if (match) {
      const level = match[1].length;
      const title = match[2].trim();
      const slug = title
        .toLowerCase()
        .replace(/[^\w\s-]/g, '')
        .replace(/\s+/g, '-')
        .replace(/-+/g, '-')
        .trim();

      toc.push({ level, title, slug });
    }
  }

  return toc;
}

/**
 * Generate search index (lowercase, no markdown formatting)
 */
function generateSearchIndex(markdown) {
  return markdown
    .toLowerCase()
    .replace(/```[\s\S]*?```/g, '') // Remove code blocks
    .replace(/`[^`]+`/g, '')         // Remove inline code
    .replace(/[#*_\[\]()]/g, '')     // Remove markdown formatting
    .replace(/\n+/g, '\n')           // Collapse multiple newlines
    .trim();
}

/**
 * Migrate a single document
 */
async function migrateDocument(docInfo, db) {
  console.log(`\nMigrating: ${docInfo.title}`);

  // Read markdown file
  const markdownPath = path.join(__dirname, '..', docInfo.file);
  const markdown = await fs.readFile(markdownPath, 'utf-8');

  // Convert to HTML
  const html = marked.parse(markdown);

  // Generate ToC
  const toc = generateToC(markdown);
  console.log(`  ✓ Generated ToC (${toc.length} headings)`);

  // Generate search index
  const searchIndex = generateSearchIndex(markdown);

  // Create document
  const document = {
    title: docInfo.title,
    slug: docInfo.slug,
    quadrant: docInfo.quadrant,
    persistence: docInfo.persistence,
    content_html: html,
    content_markdown: markdown,
    toc: toc,
    metadata: {
      author: 'System',
      date_created: new Date(),
      date_updated: new Date(),
      version: '1.0',
      document_code: null,
      related_documents: [],
      tags: docInfo.tags,
      category: docInfo.category,
      audience: docInfo.audience,
      description: docInfo.description
    },
    translations: {},
    search_index: searchIndex
  };

  // Check if document already exists
  const existing = await db.collection('documents').findOne({ slug: docInfo.slug });

  if (existing) {
    console.log(`  ⚠ Document already exists, updating...`);
    await db.collection('documents').updateOne(
      { slug: docInfo.slug },
      { $set: document }
    );
    console.log(`  ✓ Updated`);
  } else {
    await db.collection('documents').insertOne(document);
    console.log(`  ✓ Inserted`);
  }

  return docInfo.slug;
}

/**
 * Main migration function
 */
async function main() {
  console.log('=== Value Pluralism Documents Migration ===\n');

  let client;

  try {
    // Connect to MongoDB
    console.log('Connecting to MongoDB...');
    client = await mongoose.connect(config.mongodb.uri, config.mongodb.options);
    const db = mongoose.connection.db;
    console.log('✓ Connected\n');

    // Migrate each document
    const migratedSlugs = [];
    for (const docInfo of documentsToMigrate) {
      const slug = await migrateDocument(docInfo, db);
      migratedSlugs.push(slug);
    }

    // Summary
    console.log('\n=== Migration Complete ===\n');
    console.log(`✓ Migrated ${migratedSlugs.length} documents:`);
    migratedSlugs.forEach(slug => console.log(`  - ${slug}`));
    console.log('\nDocuments are now available in docs.html');
    console.log('PDFs available at:');
    migratedSlugs.forEach(slug =>
      console.log(`  - /downloads/${slug}.pdf`)
    );

  } catch (error) {
    console.error('\n✗ Migration failed:', error.message);
    console.error(error.stack);
    process.exit(1);
  } finally {
    if (client) {
      await mongoose.connection.close();
      console.log('\n✓ Database connection closed');
    }
  }
}

// Run migration
if (require.main === module) {
  main();
}

module.exports = { migrateDocument, generateToC, generateSearchIndex };