From f8ef2128fc634f5721afda82930b3a97c8edbb9d Mon Sep 17 00:00:00 2001 From: TheFlow Date: Sun, 19 Oct 2025 13:49:21 +1300 Subject: [PATCH] refactor(data): migrate legacy public field to modern visibility field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SUMMARY: Completed migration from deprecated 'public: true/false' field to modern 'visibility' field across entire codebase. Ensures single source of truth for document visibility state. MIGRATION EXECUTION: ✓ Created migration script with dry-run support ✓ Migrated 120 documents in database (removed deprecated field) ✓ Post-migration: 0 documents with 'public' field, 127 with 'visibility' ✓ Zero data loss - all documents already had visibility set correctly CODE CHANGES: 1. Database Migration (scripts/migrate-public-to-visibility.js): - Created safe migration with dry-run mode - Handles documents with both fields (cleanup) - Post-migration verification built-in - Execution: node scripts/migrate-public-to-visibility.js --execute 2. Document Model (src/models/Document.model.js): - Removed 'public' field from create() method - Updated findByQuadrant() to use visibility: 'public' - Updated findByAudience() to use visibility: 'public' - Updated search() to use visibility: 'public' 3. API Controller (src/controllers/documents.controller.js): - Removed legacy filter: { public: true, visibility: { $exists: false } } - listDocuments() now uses clean filter: visibility: 'public' - searchDocuments() now uses clean filter: visibility: 'public' 4. Scripts Updated: - upload-document.js: Removed public: true - seed-architectural-safeguards-document.js: Removed public: true - import-5-archives.js: Removed public: true - verify-34-documents.js: Updated query filter to use visibility - query-all-documents.js: Updated query filter to use visibility VERIFICATION: ✓ 0 remaining 'public: true/false' usages in src/ and scripts/ ✓ All documents use visibility field exclusively ✓ API queries now filter on visibility only ✓ Backward compatibility code removed DATA MODEL: Before: { public: true, visibility: 'public' } (redundant) After: { visibility: 'public' } (single source of truth) BENEFITS: - Cleaner data model - Single source of truth for visibility - Simplified API logic - Removed backward compatibility overhead - Consistent with document security model FRAMEWORK COMPLIANCE: Addresses SCHEDULED_TASKS.md item "Legacy public Field Migration" Completes Sprint 2 Medium Priority task NEXT STEPS (Optional): - Deploy migration to production - Monitor for any edge cases - Consider adding visibility to database indexes 🤖 Generated with Claude Code (https://claude.com/claude-code) Co-Authored-By: Claude --- scripts/import-5-archives.js | 1 - scripts/migrate-public-to-visibility.js | 138 ++++++++++++++++++ scripts/query-all-documents.js | 5 +- .../seed-architectural-safeguards-document.js | 1 - scripts/upload-document.js | 1 - scripts/verify-34-documents.js | 6 +- src/controllers/documents.controller.js | 10 +- src/models/Document.model.js | 7 +- 8 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 scripts/migrate-public-to-visibility.js diff --git a/scripts/import-5-archives.js b/scripts/import-5-archives.js index cc3a9bfa..549d680e 100644 --- a/scripts/import-5-archives.js +++ b/scripts/import-5-archives.js @@ -121,7 +121,6 @@ async function importDocument(fileInfo) { content_html, content_markdown: content, toc, - public: true, metadata: { author: frontMatter.author || 'John Stroh', version: frontMatter.version || '1.0', diff --git a/scripts/migrate-public-to-visibility.js b/scripts/migrate-public-to-visibility.js new file mode 100644 index 00000000..0513732f --- /dev/null +++ b/scripts/migrate-public-to-visibility.js @@ -0,0 +1,138 @@ +#!/usr/bin/env node + +/** + * Migrate legacy 'public' field to modern 'visibility' field + * + * SECURITY: Safe migration with dry-run support + * - Migrates public: true → visibility: 'public' + * - Migrates public: false → visibility: 'internal' + * - Preserves documents that already have visibility set + * - Removes the deprecated 'public' field after migration + */ + +const { getCollection } = require('../src/utils/db.util'); + +async function migrate(dryRun = false) { + try { + const collection = await getCollection('documents'); + + // Find documents with public field but no visibility + const docsWithPublicOnly = await collection.find({ + public: { $exists: true }, + visibility: { $exists: false } + }).toArray(); + + // Find documents with both fields (inconsistent state) + const docsWithBoth = await collection.find({ + public: { $exists: true }, + visibility: { $exists: true } + }).toArray(); + + console.log('\n📊 Migration Analysis:'); + console.log(` Documents with only 'public' field: ${docsWithPublicOnly.length}`); + console.log(` Documents with both fields: ${docsWithBoth.length}`); + console.log(` Total to migrate: ${docsWithPublicOnly.length + docsWithBoth.length}`); + + if (docsWithPublicOnly.length === 0 && docsWithBoth.length === 0) { + console.log('\n✅ No documents need migration. All documents already use visibility field.'); + return; + } + + if (dryRun) { + console.log('\n🔍 DRY RUN - No changes will be made\n'); + + // Show what would be migrated + if (docsWithPublicOnly.length > 0) { + console.log('Documents with only public field:'); + docsWithPublicOnly.forEach(doc => { + const newVisibility = doc.public ? 'public' : 'internal'; + console.log(` - ${doc.title} (${doc.slug})`); + console.log(` public: ${doc.public} → visibility: '${newVisibility}'`); + }); + } + + if (docsWithBoth.length > 0) { + console.log('\nDocuments with both fields (will remove public):'); + docsWithBoth.forEach(doc => { + console.log(` - ${doc.title} (${doc.slug})`); + console.log(` current: public=${doc.public}, visibility='${doc.visibility}'`); + console.log(` action: Keep visibility='${doc.visibility}', remove public field`); + }); + } + + console.log('\n💡 Run with --execute to perform migration'); + return; + } + + // Perform actual migration + console.log('\n🔄 Performing migration...\n'); + + let migratedCount = 0; + + // Migrate documents with only public field + for (const doc of docsWithPublicOnly) { + const visibility = doc.public ? 'public' : 'internal'; + + await collection.updateOne( + { _id: doc._id }, + { + $set: { visibility }, + $unset: { public: "" } + } + ); + + console.log(`✓ ${doc.title}: public=${doc.public} → visibility='${visibility}'`); + migratedCount++; + } + + // Clean up documents with both fields (keep visibility, remove public) + for (const doc of docsWithBoth) { + await collection.updateOne( + { _id: doc._id }, + { $unset: { public: "" } } + ); + + console.log(`✓ ${doc.title}: Removed public field, kept visibility='${doc.visibility}'`); + migratedCount++; + } + + console.log(`\n✅ Migration complete! ${migratedCount} documents updated.`); + + // Verify results + const remainingWithPublic = await collection.countDocuments({ public: { $exists: true } }); + const totalWithVisibility = await collection.countDocuments({ visibility: { $exists: true } }); + + console.log('\n📊 Post-migration verification:'); + console.log(` Documents with 'public' field: ${remainingWithPublic}`); + console.log(` Documents with 'visibility' field: ${totalWithVisibility}`); + + if (remainingWithPublic > 0) { + console.warn('\n⚠️ Warning: Some documents still have the public field. Review manually.'); + } else { + console.log('\n✅ All documents successfully migrated to visibility field!'); + } + + } catch (error) { + console.error('❌ Migration failed:', error); + throw error; + } +} + +// CLI interface +const args = process.argv.slice(2); +const dryRun = !args.includes('--execute'); + +if (dryRun) { + console.log('🔍 Running in DRY RUN mode (no changes will be made)'); + console.log(' Use --execute flag to perform actual migration\n'); +} + +migrate(dryRun) + .then(() => { + console.log('\n✨ Script complete'); + process.exit(0); + }) + .catch((error) => { + console.error('\n💥 Script failed:', error); + process.exit(1); + }); diff --git a/scripts/query-all-documents.js b/scripts/query-all-documents.js index 6771069d..f879ea67 100644 --- a/scripts/query-all-documents.js +++ b/scripts/query-all-documents.js @@ -9,10 +9,7 @@ async function queryAllDocuments() { const collection = db.collection('documents'); const documents = await collection.find({ - $or: [ - { visibility: 'public' }, - { public: true, visibility: { $exists: false } } - ] + visibility: 'public' }) .sort({ category: 1, order: 1, 'metadata.date_created': -1 }) .toArray(); diff --git a/scripts/seed-architectural-safeguards-document.js b/scripts/seed-architectural-safeguards-document.js index 9ff3b63a..85d7b0db 100644 --- a/scripts/seed-architectural-safeguards-document.js +++ b/scripts/seed-architectural-safeguards-document.js @@ -59,7 +59,6 @@ async function seedDocument() { content_html: htmlContent, content_markdown: rawContent, toc: tableOfContents, - public: true, security_classification: { contains_credentials: false, contains_financial_info: false, diff --git a/scripts/upload-document.js b/scripts/upload-document.js index 61f59288..ccdc12ba 100644 --- a/scripts/upload-document.js +++ b/scripts/upload-document.js @@ -512,7 +512,6 @@ async function uploadDocument() { content_html: htmlContent, content_markdown: rawContent, toc: tableOfContents, - public: true, security_classification: { contains_credentials: false, contains_financial_info: false, diff --git a/scripts/verify-34-documents.js b/scripts/verify-34-documents.js index e2f1c32f..47a9f10d 100644 --- a/scripts/verify-34-documents.js +++ b/scripts/verify-34-documents.js @@ -19,11 +19,7 @@ async function verify34Documents() { // Get documents with proper categories (not 'none') const documents = await collection.find({ - $or: [ - { visibility: 'public' }, - { visibility: 'archived' }, - { public: true, visibility: { $exists: false } } - ], + visibility: { $in: ['public', 'archived'] }, category: { $in: ['getting-started', 'technical-reference', 'research-theory', 'advanced-topics', 'case-studies', 'business-leadership', 'archives'] } }) .sort({ category: 1, order: 1 }) diff --git a/src/controllers/documents.controller.js b/src/controllers/documents.controller.js index 49d81592..f74c08ef 100644 --- a/src/controllers/documents.controller.js +++ b/src/controllers/documents.controller.js @@ -20,10 +20,7 @@ async function listDocuments(req, res) { // Build filter - only show public documents (not internal/confidential) const filter = { - $or: [ - { visibility: 'public' }, - { public: true, visibility: { $exists: false } } // Legacy support - ] + visibility: 'public' }; if (quadrant) { filter.quadrant = quadrant; @@ -122,10 +119,7 @@ async function searchDocuments(req, res) { // Build filter for faceted search const filter = { - $or: [ - { visibility: 'public' }, - { public: true, visibility: { $exists: false } } // Legacy support - ] + visibility: 'public' }; // Add facet filters diff --git a/src/models/Document.model.js b/src/models/Document.model.js index 2532b8a9..e1a39fcf 100644 --- a/src/models/Document.model.js +++ b/src/models/Document.model.js @@ -44,7 +44,6 @@ class Document { content_html: data.content_html, content_markdown: data.content_markdown, toc: data.toc || [], - public: data.public !== undefined ? data.public : true, // Deprecated - use visibility instead security_classification: data.security_classification || { contains_credentials: false, contains_financial_info: false, @@ -95,7 +94,7 @@ class Document { const filter = { quadrant }; if (publicOnly) { - filter.public = true; + filter.visibility = 'public'; } return await collection @@ -115,7 +114,7 @@ class Document { const filter = { audience }; if (publicOnly) { - filter.public = true; + filter.visibility = 'public'; } return await collection @@ -135,7 +134,7 @@ class Document { const filter = { $text: { $search: query } }; if (publicOnly) { - filter.public = true; + filter.visibility = 'public'; } return await collection