refactor(data): migrate legacy public field to modern visibility field

SUMMARY: Completed migration from deprecated 'public: true/false' field to modern 'visibility' field across entire codebase. Ensures single source of truth for document visibility state. MIGRATION EXECUTION: ✓ Created migration script with dry-run support ✓ Migrated 120 documents in database (removed deprecated field) ✓ Post-migration: 0 documents with 'public' field, 127 with 'visibility' ✓ Zero data loss - all documents already had visibility set correctly CODE CHANGES: 1. Database Migration (scripts/migrate-public-to-visibility.js): - Created safe migration with dry-run mode - Handles documents with both fields (cleanup) - Post-migration verification built-in - Execution: node scripts/migrate-public-to-visibility.js --execute 2. Document Model (src/models/Document.model.js): - Removed 'public' field from create() method - Updated findByQuadrant() to use visibility: 'public' - Updated findByAudience() to use visibility: 'public' - Updated search() to use visibility: 'public' 3. API Controller (src/controllers/documents.controller.js): - Removed legacy filter: { public: true, visibility: { $exists: false } } - listDocuments() now uses clean filter: visibility: 'public' - searchDocuments() now uses clean filter: visibility: 'public' 4. Scripts Updated: - upload-document.js: Removed public: true - seed-architectural-safeguards-document.js: Removed public: true - import-5-archives.js: Removed public: true - verify-34-documents.js: Updated query filter to use visibility - query-all-documents.js: Updated query filter to use visibility VERIFICATION: ✓ 0 remaining 'public: true/false' usages in src/ and scripts/ ✓ All documents use visibility field exclusively ✓ API queries now filter on visibility only ✓ Backward compatibility code removed DATA MODEL: Before: { public: true, visibility: 'public' } (redundant) After: { visibility: 'public' } (single source of truth) BENEFITS: - Cleaner data model - Single source of truth for visibility - Simplified API logic - Removed backward compatibility overhead - Consistent with document security model FRAMEWORK COMPLIANCE: Addresses SCHEDULED_TASKS.md item "Legacy public Field Migration" Completes Sprint 2 Medium Priority task NEXT STEPS (Optional): - Deploy migration to production - Monitor for any edge cases - Consider adding visibility to database indexes 🤖 Generated with Claude Code (https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-19 13:49:21 +13:00 · 2025-10-19 13:49:21 +13:00 · 13b6910198
commit 13b6910198
parent 69c12a785b
9 changed files with 281 additions and 27 deletions
--- a/.claude/metrics/hooks-metrics.json
+++ b/.claude/metrics/hooks-metrics.json
@ -3422,6 +3422,139 @@
      "file": "/home/theflow/projects/tractatus/public/js/admin/dashboard.js",
      "result": "passed",
      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:09.018Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:16.268Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:21.086Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:26.796Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:40.866Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:49.535Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:43:55.424Z",
+      "file": "/home/theflow/projects/tractatus/SCHEDULED_TASKS.md",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-write",
+      "timestamp": "2025-10-19T00:45:05.788Z",
+      "file": "/home/theflow/projects/tractatus/scripts/migrate-public-to-visibility.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:46:27.199Z",
+      "file": "/home/theflow/projects/tractatus/src/models/Document.model.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:46:33.940Z",
+      "file": "/home/theflow/projects/tractatus/src/models/Document.model.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:46:39.642Z",
+      "file": "/home/theflow/projects/tractatus/src/models/Document.model.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:46:46.547Z",
+      "file": "/home/theflow/projects/tractatus/src/models/Document.model.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:47:05.109Z",
+      "file": "/home/theflow/projects/tractatus/src/controllers/documents.controller.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:47:18.592Z",
+      "file": "/home/theflow/projects/tractatus/src/controllers/documents.controller.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:47:42.966Z",
+      "file": "/home/theflow/projects/tractatus/scripts/upload-document.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:47:58.005Z",
+      "file": "/home/theflow/projects/tractatus/scripts/seed-architectural-safeguards-document.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:48:18.766Z",
+      "file": "/home/theflow/projects/tractatus/scripts/import-5-archives.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:48:30.645Z",
+      "file": "/home/theflow/projects/tractatus/scripts/verify-34-documents.js",
+      "result": "passed",
+      "reason": null
+    },
+    {
+      "hook": "validate-file-edit",
+      "timestamp": "2025-10-19T00:48:39.023Z",
+      "file": "/home/theflow/projects/tractatus/scripts/query-all-documents.js",
+      "result": "passed",
+      "reason": null
    }
  ],
  "blocks": [
@ -3643,10 +3776,10 @@
    }
  ],
  "session_stats": {
-    "total_edit_hooks": 319,
+    "total_edit_hooks": 337,
    "total_edit_blocks": 32,
-    "last_updated": "2025-10-19T00:41:10.410Z",
-    "total_write_hooks": 170,
+    "last_updated": "2025-10-19T00:48:39.023Z",
+    "total_write_hooks": 171,
    "total_write_blocks": 4
  }
 }
--- a/scripts/import-5-archives.js
+++ b/scripts/import-5-archives.js
@ -121,7 +121,6 @@ async function importDocument(fileInfo) {
      content_html,
      content_markdown: content,
      toc,
-      public: true,
      metadata: {
        author: frontMatter.author || 'John Stroh',
        version: frontMatter.version || '1.0',
--- a/scripts/migrate-public-to-visibility.js
+++ b/scripts/migrate-public-to-visibility.js
@ -0,0 +1,138 @@
+#!/usr/bin/env node
+
+/**
+ * Migrate legacy 'public' field to modern 'visibility' field
+ *
+ * SECURITY: Safe migration with dry-run support
+ * - Migrates public: true → visibility: 'public'
+ * - Migrates public: false → visibility: 'internal'
+ * - Preserves documents that already have visibility set
+ * - Removes the deprecated 'public' field after migration
+ */
+
+const { getCollection } = require('../src/utils/db.util');
+
+async function migrate(dryRun = false) {
+  try {
+    const collection = await getCollection('documents');
+
+    // Find documents with public field but no visibility
+    const docsWithPublicOnly = await collection.find({
+      public: { $exists: true },
+      visibility: { $exists: false }
+    }).toArray();
+
+    // Find documents with both fields (inconsistent state)
+    const docsWithBoth = await collection.find({
+      public: { $exists: true },
+      visibility: { $exists: true }
+    }).toArray();
+
+    console.log('\n📊 Migration Analysis:');
+    console.log(`   Documents with only 'public' field: ${docsWithPublicOnly.length}`);
+    console.log(`   Documents with both fields: ${docsWithBoth.length}`);
+    console.log(`   Total to migrate: ${docsWithPublicOnly.length + docsWithBoth.length}`);
+
+    if (docsWithPublicOnly.length === 0 && docsWithBoth.length === 0) {
+      console.log('\n✅ No documents need migration. All documents already use visibility field.');
+      return;
+    }
+
+    if (dryRun) {
+      console.log('\n🔍 DRY RUN - No changes will be made\n');
+
+      // Show what would be migrated
+      if (docsWithPublicOnly.length > 0) {
+        console.log('Documents with only public field:');
+        docsWithPublicOnly.forEach(doc => {
+          const newVisibility = doc.public ? 'public' : 'internal';
+          console.log(`   - ${doc.title} (${doc.slug})`);
+          console.log(`     public: ${doc.public} → visibility: '${newVisibility}'`);
+        });
+      }
+
+      if (docsWithBoth.length > 0) {
+        console.log('\nDocuments with both fields (will remove public):');
+        docsWithBoth.forEach(doc => {
+          console.log(`   - ${doc.title} (${doc.slug})`);
+          console.log(`     current: public=${doc.public}, visibility='${doc.visibility}'`);
+          console.log(`     action: Keep visibility='${doc.visibility}', remove public field`);
+        });
+      }
+
+      console.log('\n💡 Run with --execute to perform migration');
+      return;
+    }
+
+    // Perform actual migration
+    console.log('\n🔄 Performing migration...\n');
+
+    let migratedCount = 0;
+
+    // Migrate documents with only public field
+    for (const doc of docsWithPublicOnly) {
+      const visibility = doc.public ? 'public' : 'internal';
+
+      await collection.updateOne(
+        { _id: doc._id },
+        {
+          $set: { visibility },
+          $unset: { public: "" }
+        }
+      );
+
+      console.log(`✓ ${doc.title}: public=${doc.public} → visibility='${visibility}'`);
+      migratedCount++;
+    }
+
+    // Clean up documents with both fields (keep visibility, remove public)
+    for (const doc of docsWithBoth) {
+      await collection.updateOne(
+        { _id: doc._id },
+        { $unset: { public: "" } }
+      );
+
+      console.log(`✓ ${doc.title}: Removed public field, kept visibility='${doc.visibility}'`);
+      migratedCount++;
+    }
+
+    console.log(`\n✅ Migration complete! ${migratedCount} documents updated.`);
+
+    // Verify results
+    const remainingWithPublic = await collection.countDocuments({ public: { $exists: true } });
+    const totalWithVisibility = await collection.countDocuments({ visibility: { $exists: true } });
+
+    console.log('\n📊 Post-migration verification:');
+    console.log(`   Documents with 'public' field: ${remainingWithPublic}`);
+    console.log(`   Documents with 'visibility' field: ${totalWithVisibility}`);
+
+    if (remainingWithPublic > 0) {
+      console.warn('\n⚠️  Warning: Some documents still have the public field. Review manually.');
+    } else {
+      console.log('\n✅ All documents successfully migrated to visibility field!');
+    }
+
+  } catch (error) {
+    console.error('❌ Migration failed:', error);
+    throw error;
+  }
+}
+
+// CLI interface
+const args = process.argv.slice(2);
+const dryRun = !args.includes('--execute');
+
+if (dryRun) {
+  console.log('🔍 Running in DRY RUN mode (no changes will be made)');
+  console.log('   Use --execute flag to perform actual migration\n');
+}
+
+migrate(dryRun)
+  .then(() => {
+    console.log('\n✨ Script complete');
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error('\n💥 Script failed:', error);
+    process.exit(1);
+  });
--- a/scripts/query-all-documents.js
+++ b/scripts/query-all-documents.js
@ -9,10 +9,7 @@ async function queryAllDocuments() {
    const collection = db.collection('documents');

    const documents = await collection.find({
-      $or: [
-        { visibility: 'public' },
-        { public: true, visibility: { $exists: false } }
-      ]
+      visibility: 'public'
    })
    .sort({ category: 1, order: 1, 'metadata.date_created': -1 })
    .toArray();
--- a/scripts/seed-architectural-safeguards-document.js
+++ b/scripts/seed-architectural-safeguards-document.js
@ -59,7 +59,6 @@ async function seedDocument() {
      content_html: htmlContent,
      content_markdown: rawContent,
      toc: tableOfContents,
-      public: true,
      security_classification: {
        contains_credentials: false,
        contains_financial_info: false,
--- a/scripts/upload-document.js
+++ b/scripts/upload-document.js
@ -512,7 +512,6 @@ async function uploadDocument() {
      content_html: htmlContent,
      content_markdown: rawContent,
      toc: tableOfContents,
-      public: true,
      security_classification: {
        contains_credentials: false,
        contains_financial_info: false,
--- a/scripts/verify-34-documents.js
+++ b/scripts/verify-34-documents.js
@ -19,11 +19,7 @@ async function verify34Documents() {

    // Get documents with proper categories (not 'none')
    const documents = await collection.find({
-      $or: [
-        { visibility: 'public' },
-        { visibility: 'archived' },
-        { public: true, visibility: { $exists: false } }
-      ],
+      visibility: { $in: ['public', 'archived'] },
      category: { $in: ['getting-started', 'technical-reference', 'research-theory', 'advanced-topics', 'case-studies', 'business-leadership', 'archives'] }
    })
    .sort({ category: 1, order: 1 })
--- a/src/controllers/documents.controller.js
+++ b/src/controllers/documents.controller.js
@ -20,10 +20,7 @@ async function listDocuments(req, res) {

    // Build filter - only show public documents (not internal/confidential)
    const filter = {
-      $or: [
-        { visibility: 'public' },
-        { public: true, visibility: { $exists: false } } // Legacy support
-      ]
+      visibility: 'public'
    };
    if (quadrant) {
      filter.quadrant = quadrant;
@ -122,10 +119,7 @@ async function searchDocuments(req, res) {

    // Build filter for faceted search
    const filter = {
-      $or: [
-        { visibility: 'public' },
-        { public: true, visibility: { $exists: false } } // Legacy support
-      ]
+      visibility: 'public'
    };

    // Add facet filters
--- a/src/models/Document.model.js
+++ b/src/models/Document.model.js
@ -44,7 +44,6 @@ class Document {
      content_html: data.content_html,
      content_markdown: data.content_markdown,
      toc: data.toc || [],
-      public: data.public !== undefined ? data.public : true, // Deprecated - use visibility instead
      security_classification: data.security_classification || {
        contains_credentials: false,
        contains_financial_info: false,
@ -95,7 +94,7 @@ class Document {

    const filter = { quadrant };
    if (publicOnly) {
-      filter.public = true;
+      filter.visibility = 'public';
    }

    return await collection
@ -115,7 +114,7 @@ class Document {

    const filter = { audience };
    if (publicOnly) {
-      filter.public = true;
+      filter.visibility = 'public';
    }

    return await collection
@ -135,7 +134,7 @@ class Document {

    const filter = { $text: { $search: query } };
    if (publicOnly) {
-      filter.public = true;
+      filter.visibility = 'public';
    }

    return await collection