test: add comprehensive unit test suite for Tractatus governance services
Implemented comprehensive unit test coverage for all 5 core governance services: 1. InstructionPersistenceClassifier.test.js (51 tests) - Quadrant classification (STR/OPS/TAC/SYS/STO) - Persistence level calculation - Verification requirements - Temporal scope detection - Explicitness measurement - 27027 failure mode prevention - Metadata preservation - Edge cases and consistency 2. CrossReferenceValidator.test.js (39 tests) - 27027 failure mode prevention (critical) - Conflict detection between actions and instructions - Relevance calculation and prioritization - Conflict severity levels (CRITICAL/WARNING/MINOR) - Parameter extraction from actions/instructions - Lookback window management - Complex multi-parameter scenarios 3. BoundaryEnforcer.test.js (39 tests) - Tractatus 12.1-12.7 boundary enforcement - VALUES, WISDOM, AGENCY, PURPOSE boundaries - Human judgment requirements - Multi-boundary violation detection - Safe AI operations (allowed vs restricted) - Context-aware enforcement - Audit trail generation 4. ContextPressureMonitor.test.js (32 tests) - Token usage pressure detection - Conversation length monitoring - Task complexity analysis - Error frequency tracking - Pressure level calculation (NORMAL→DANGEROUS) - Recommendations by pressure level - 27027 incident correlation - Pressure history and trends 5. MetacognitiveVerifier.test.js (31 tests) - Alignment verification (action vs reasoning) - Coherence checking (internal consistency) - Completeness verification - Safety assessment and risk levels - Alternative consideration - Confidence calculation - Pressure-adjusted verification - 27027 failure mode prevention Total: 192 tests (30 currently passing) Test Status: - Tests define expected API for all governance services - 30/192 tests passing with current service implementations - Failing tests identify missing methods (getStats, reset, etc.) - Comprehensive test coverage guides future development - All tests use correct singleton pattern for service instances Next Steps: - Implement missing service methods (getStats, reset, etc.) - Align service return structures with test expectations - Add integration tests for governance middleware - Achieve >80% test pass rate The test suite provides a world-class specification for the Tractatus governance framework and ensures AI safety guarantees are testable. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TheFlow
parent
2193b46a52
commit
e8cc023a05
5 changed files with 2799 additions and 0 deletions
579
tests/unit/BoundaryEnforcer.test.js
Normal file
579
tests/unit/BoundaryEnforcer.test.js
Normal file
|
|
@ -0,0 +1,579 @@
|
|||
/**
|
||||
* Unit Tests for BoundaryEnforcer
|
||||
* Tests Tractatus philosophical boundaries (12.1-12.7) enforcement
|
||||
*/
|
||||
|
||||
const enforcer = require('../../src/services/BoundaryEnforcer.service');
|
||||
|
||||
describe('BoundaryEnforcer', () => {
|
||||
beforeEach(() => {
|
||||
// Enforcer is a singleton instance
|
||||
});
|
||||
|
||||
describe('Tractatus 12.1 - Values Boundary', () => {
|
||||
test('should require human judgment for values decisions', () => {
|
||||
const decision = {
|
||||
type: 'values_change',
|
||||
description: 'Change privacy policy to prioritize convenience',
|
||||
domain: 'values'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('VALUES');
|
||||
expect(result.tractatus_section).toBe('12.1');
|
||||
expect(result.reason).toContain('Values cannot be automated');
|
||||
});
|
||||
|
||||
test('should block AI from making values trade-offs', () => {
|
||||
const decision = {
|
||||
type: 'trade_off',
|
||||
description: 'Trade user privacy for performance',
|
||||
involves_values: true
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
});
|
||||
|
||||
test('should allow AI to verify alignment with existing values', () => {
|
||||
const decision = {
|
||||
type: 'verify_alignment',
|
||||
description: 'Check if action aligns with privacy values',
|
||||
domain: 'verification'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
expect(result.human_required).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tractatus 12.2 - Innovation Boundary', () => {
|
||||
test('should require human judgment for novel architectural decisions', () => {
|
||||
const decision = {
|
||||
type: 'architecture',
|
||||
description: 'Propose entirely new approach to authentication',
|
||||
novelty: 'high',
|
||||
domain: 'innovation'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('INNOVATION');
|
||||
});
|
||||
|
||||
test('should allow AI to facilitate innovation within approved patterns', () => {
|
||||
const decision = {
|
||||
type: 'optimization',
|
||||
description: 'Optimize existing authentication flow',
|
||||
novelty: 'low'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tractatus 12.3 - Wisdom Boundary', () => {
|
||||
test('should require human judgment for strategic direction changes', () => {
|
||||
const decision = {
|
||||
type: 'strategic_direction',
|
||||
description: 'Shift product focus to new market segment',
|
||||
domain: 'wisdom'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('WISDOM');
|
||||
expect(result.tractatus_section).toBe('12.3');
|
||||
});
|
||||
|
||||
test('should allow AI to support wisdom with data analysis', () => {
|
||||
const decision = {
|
||||
type: 'data_analysis',
|
||||
description: 'Analyze market trends to inform strategy',
|
||||
domain: 'support'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tractatus 12.4 - Purpose Boundary', () => {
|
||||
test('should require human judgment for defining project purpose', () => {
|
||||
const decision = {
|
||||
type: 'define_purpose',
|
||||
description: 'Define the core mission of the project',
|
||||
domain: 'purpose'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('PURPOSE');
|
||||
});
|
||||
|
||||
test('should allow AI to preserve existing purpose in implementations', () => {
|
||||
const decision = {
|
||||
type: 'implementation',
|
||||
description: 'Implement feature according to stated purpose',
|
||||
domain: 'preservation'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tractatus 12.5 - Meaning Boundary', () => {
|
||||
test('should require human judgment for determining significance', () => {
|
||||
const decision = {
|
||||
type: 'significance',
|
||||
description: 'Decide what makes this project meaningful',
|
||||
domain: 'meaning'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('MEANING');
|
||||
});
|
||||
|
||||
test('should allow AI to recognize patterns related to meaning', () => {
|
||||
const decision = {
|
||||
type: 'pattern_recognition',
|
||||
description: 'Identify user engagement patterns',
|
||||
domain: 'recognition'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tractatus 12.6 - Agency Boundary', () => {
|
||||
test('should require human judgment for decisions affecting human agency', () => {
|
||||
const decision = {
|
||||
type: 'agency_affecting',
|
||||
description: 'Automatically approve user requests',
|
||||
domain: 'agency',
|
||||
affects_human_choice: true
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('AGENCY');
|
||||
expect(result.tractatus_section).toBe('12.6');
|
||||
});
|
||||
|
||||
test('should allow AI to respect human agency through notification', () => {
|
||||
const decision = {
|
||||
type: 'notify',
|
||||
description: 'Notify user of available options',
|
||||
respects_agency: true
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Boundary Detection', () => {
|
||||
test('should detect values-related keywords in decisions', () => {
|
||||
const decision = {
|
||||
description: 'Change our core values to prioritize speed'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('VALUES');
|
||||
});
|
||||
|
||||
test('should detect wisdom-related keywords', () => {
|
||||
const decision = {
|
||||
description: 'Make strategic decision about company direction'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(['WISDOM', 'VALUES']).toContain(result.boundary);
|
||||
});
|
||||
|
||||
test('should detect agency-affecting keywords', () => {
|
||||
const decision = {
|
||||
description: 'Remove user choice and automate approval'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.boundary).toBe('AGENCY');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Multi-Boundary Violations', () => {
|
||||
test('should detect when decision crosses multiple boundaries', () => {
|
||||
const decision = {
|
||||
type: 'major_change',
|
||||
description: 'Redefine project purpose and change core values',
|
||||
domain: ['purpose', 'values']
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.violated_boundaries.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
test('should require most restrictive boundary when multiple apply', () => {
|
||||
const decision = {
|
||||
description: 'Strategic values decision affecting user agency'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.violated_boundaries.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Safe AI Operations', () => {
|
||||
test('should allow technical implementation decisions', () => {
|
||||
const decision = {
|
||||
type: 'technical',
|
||||
description: 'Optimize database query performance',
|
||||
domain: 'system'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
expect(result.human_required).toBe(false);
|
||||
});
|
||||
|
||||
test('should allow code generation within approved patterns', () => {
|
||||
const decision = {
|
||||
type: 'code_generation',
|
||||
description: 'Generate CRUD endpoints following existing patterns',
|
||||
domain: 'system'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should allow data analysis and recommendations', () => {
|
||||
const decision = {
|
||||
type: 'analysis',
|
||||
description: 'Analyze logs and recommend optimizations',
|
||||
domain: 'support'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should allow documentation improvements', () => {
|
||||
const decision = {
|
||||
type: 'documentation',
|
||||
description: 'Improve API documentation clarity',
|
||||
domain: 'support'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Boundary Enforcement with Context', () => {
|
||||
test('should consider user role in enforcement', () => {
|
||||
const decision = {
|
||||
type: 'values_change',
|
||||
description: 'Update privacy policy',
|
||||
domain: 'values'
|
||||
};
|
||||
|
||||
const context = {
|
||||
user_role: 'admin',
|
||||
approval_status: 'approved'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision, context);
|
||||
|
||||
// Still requires human judgment, but context is noted
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.context).toBeDefined();
|
||||
});
|
||||
|
||||
test('should escalate dangerous operations regardless of context', () => {
|
||||
const decision = {
|
||||
type: 'values_change',
|
||||
description: 'Completely rewrite core values',
|
||||
domain: 'values'
|
||||
};
|
||||
|
||||
const context = {
|
||||
pressure_level: 'CRITICAL'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision, context);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
expect(result.escalation_required).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Boundary Explanations', () => {
|
||||
test('should provide clear explanation for VALUES boundary', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Change privacy settings'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.reason).toContain('Values cannot be automated');
|
||||
expect(result.explanation).toBeDefined();
|
||||
});
|
||||
|
||||
test('should provide Tractatus section reference', () => {
|
||||
const decision = {
|
||||
domain: 'agency',
|
||||
description: 'Remove user consent requirement'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.tractatus_section).toBe('12.6');
|
||||
expect(result.principle).toContain('Agency cannot be simulated');
|
||||
});
|
||||
|
||||
test('should suggest alternative approaches', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Automatically decide privacy policy'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.alternatives).toBeDefined();
|
||||
expect(result.alternatives.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
test('should handle decisions with no clear boundary', () => {
|
||||
const decision = {
|
||||
type: 'routine',
|
||||
description: 'Restart development server'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
expect(result.boundary).toBeNull();
|
||||
});
|
||||
|
||||
test('should handle null or undefined decision gracefully', () => {
|
||||
expect(() => {
|
||||
enforcer.enforce(null);
|
||||
}).not.toThrow();
|
||||
|
||||
const result = enforcer.enforce(null);
|
||||
expect(result.allowed).toBe(false);
|
||||
});
|
||||
|
||||
test('should handle decision with empty description', () => {
|
||||
const decision = {
|
||||
description: ''
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Pre-approved Exceptions', () => {
|
||||
test('should allow pre-approved operations even if boundary-adjacent', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Verify compliance with stated values',
|
||||
pre_approved: true
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
// Verification is allowed, modification is not
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should not allow modifications under pre-approval', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Modify core values',
|
||||
pre_approved: true
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(false);
|
||||
expect(result.human_required).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Audit Trail', () => {
|
||||
test('should create audit record for boundary violations', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Change security policy'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.audit_record).toBeDefined();
|
||||
expect(result.audit_record.timestamp).toBeDefined();
|
||||
expect(result.audit_record.boundary_violated).toBe('VALUES');
|
||||
});
|
||||
|
||||
test('should track all enforcement decisions', () => {
|
||||
enforcer.enforce({ type: 'test1' });
|
||||
enforcer.enforce({ type: 'test2' });
|
||||
|
||||
const stats = enforcer.getStats();
|
||||
|
||||
expect(stats.total_enforcements).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Integration with Classification', () => {
|
||||
test('should respect STRATEGIC quadrant as potential boundary violation', () => {
|
||||
const decision = {
|
||||
classification: {
|
||||
quadrant: 'STRATEGIC',
|
||||
persistence: 'HIGH'
|
||||
},
|
||||
description: 'Define long-term project direction'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.human_required).toBe(true);
|
||||
});
|
||||
|
||||
test('should allow SYSTEM and TACTICAL quadrants without boundary concerns', () => {
|
||||
const decision = {
|
||||
classification: {
|
||||
quadrant: 'SYSTEM',
|
||||
persistence: 'MEDIUM'
|
||||
},
|
||||
description: 'Optimize database indexes'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Singleton Pattern', () => {
|
||||
test('should export singleton instance with required methods', () => {
|
||||
expect(typeof enforcer.enforce).toBe('function');
|
||||
expect(typeof enforcer.getStats).toBe('function');
|
||||
});
|
||||
|
||||
test('should maintain enforcement history across calls', () => {
|
||||
const beforeCount = enforcer.getStats().total_enforcements;
|
||||
|
||||
enforcer.enforce({ type: 'test' });
|
||||
|
||||
const afterCount = enforcer.getStats().total_enforcements;
|
||||
|
||||
expect(afterCount).toBe(beforeCount + 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Statistics Tracking', () => {
|
||||
test('should track enforcement statistics', () => {
|
||||
const stats = enforcer.getStats();
|
||||
|
||||
expect(stats).toHaveProperty('total_enforcements');
|
||||
expect(stats).toHaveProperty('boundaries_violated');
|
||||
expect(stats).toHaveProperty('human_required_count');
|
||||
});
|
||||
|
||||
test('should track violations by boundary type', () => {
|
||||
enforcer.enforce({ domain: 'values', description: 'test1' });
|
||||
enforcer.enforce({ domain: 'agency', description: 'test2' });
|
||||
|
||||
const stats = enforcer.getStats();
|
||||
|
||||
expect(stats.by_boundary).toHaveProperty('VALUES');
|
||||
expect(stats.by_boundary).toHaveProperty('AGENCY');
|
||||
});
|
||||
|
||||
test('should increment enforcement count after enforce()', () => {
|
||||
const before = enforcer.getStats().total_enforcements;
|
||||
|
||||
enforcer.enforce({ type: 'test' });
|
||||
|
||||
const after = enforcer.getStats().total_enforcements;
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Safe Escalation Paths', () => {
|
||||
test('should provide escalation path for blocked decisions', () => {
|
||||
const decision = {
|
||||
domain: 'values',
|
||||
description: 'Redefine privacy policy',
|
||||
urgency: 'high'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.escalation_path).toBeDefined();
|
||||
expect(result.escalation_path).toContain('human approval');
|
||||
});
|
||||
|
||||
test('should suggest deferred execution for strategic decisions', () => {
|
||||
const decision = {
|
||||
classification: { quadrant: 'STRATEGIC' },
|
||||
description: 'Major architectural change'
|
||||
};
|
||||
|
||||
const result = enforcer.enforce(decision);
|
||||
|
||||
expect(result.suggested_action).toContain('defer');
|
||||
});
|
||||
});
|
||||
});
|
||||
548
tests/unit/ContextPressureMonitor.test.js
Normal file
548
tests/unit/ContextPressureMonitor.test.js
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
/**
|
||||
* Unit Tests for ContextPressureMonitor
|
||||
* Tests context pressure analysis and error probability detection
|
||||
*/
|
||||
|
||||
const monitor = require('../../src/services/ContextPressureMonitor.service');
|
||||
|
||||
describe('ContextPressureMonitor', () => {
|
||||
beforeEach(() => {
|
||||
// Reset monitor state if method exists
|
||||
if (monitor.reset) {
|
||||
monitor.reset();
|
||||
}
|
||||
});
|
||||
|
||||
describe('Token Usage Pressure', () => {
|
||||
test('should detect NORMAL pressure at low token usage', () => {
|
||||
const context = {
|
||||
token_usage: 0.2,
|
||||
token_limit: 200000
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.level).toBe('NORMAL');
|
||||
expect(result.metrics.tokenUsage.score).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
test('should detect ELEVATED pressure at moderate token usage', () => {
|
||||
const context = {
|
||||
token_usage: 0.55,
|
||||
token_limit: 200000
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(['ELEVATED', 'HIGH']).toContain(result.level);
|
||||
});
|
||||
|
||||
test('should detect CRITICAL pressure at high token usage', () => {
|
||||
const context = {
|
||||
token_usage: 0.85,
|
||||
token_limit: 200000
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(['HIGH', 'CRITICAL']).toContain(result.level);
|
||||
});
|
||||
|
||||
test('should detect DANGEROUS pressure near token limit', () => {
|
||||
const context = {
|
||||
token_usage: 0.95,
|
||||
token_limit: 200000
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(['CRITICAL', 'DANGEROUS']).toContain(result.level);
|
||||
expect(result.recommendations).toContain('IMMEDIATE_HALT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Conversation Length Pressure', () => {
|
||||
test('should detect NORMAL pressure for short conversations', () => {
|
||||
const context = {
|
||||
conversation_length: 10,
|
||||
messages_count: 10
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.conversationLength.score).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
test('should detect ELEVATED pressure for medium conversations', () => {
|
||||
const context = {
|
||||
conversation_length: 50,
|
||||
messages_count: 50
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.conversationLength.score).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should detect HIGH pressure for long conversations', () => {
|
||||
const context = {
|
||||
conversation_length: 100,
|
||||
messages_count: 100
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.conversationLength.score).toBeGreaterThan(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Task Complexity Pressure', () => {
|
||||
test('should detect low complexity for simple tasks', () => {
|
||||
const context = {
|
||||
task_depth: 1,
|
||||
dependencies: 0,
|
||||
file_modifications: 1
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.taskComplexity.score).toBeLessThan(0.3);
|
||||
});
|
||||
|
||||
test('should detect high complexity for multi-step tasks', () => {
|
||||
const context = {
|
||||
task_depth: 5,
|
||||
dependencies: 10,
|
||||
file_modifications: 15,
|
||||
concurrent_operations: 8
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.taskComplexity.score).toBeGreaterThan(0.5);
|
||||
});
|
||||
|
||||
test('should consider nested sub-tasks in complexity', () => {
|
||||
const context = {
|
||||
task_depth: 3,
|
||||
subtasks_pending: 12,
|
||||
dependencies: 8
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.taskComplexity).toBeDefined();
|
||||
expect(result.metrics.taskComplexity.factors).toContain('high task depth');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Frequency Pressure', () => {
|
||||
test('should detect NORMAL with no recent errors', () => {
|
||||
const context = {
|
||||
errors_recent: 0,
|
||||
errors_last_hour: 0
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.errorFrequency.score).toBe(0);
|
||||
});
|
||||
|
||||
test('should detect ELEVATED with occasional errors', () => {
|
||||
const context = {
|
||||
errors_recent: 2,
|
||||
errors_last_hour: 2
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.errorFrequency.score).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should detect CRITICAL with frequent errors', () => {
|
||||
const context = {
|
||||
errors_recent: 10,
|
||||
errors_last_hour: 10,
|
||||
error_pattern: 'repeating'
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.errorFrequency.score).toBeGreaterThan(0.7);
|
||||
expect(result.level).toMatch(/HIGH|CRITICAL|DANGEROUS/);
|
||||
});
|
||||
|
||||
test('should track error patterns over time', () => {
|
||||
// Simulate increasing error rate
|
||||
monitor.recordError({ type: 'syntax_error' });
|
||||
monitor.recordError({ type: 'syntax_error' });
|
||||
monitor.recordError({ type: 'syntax_error' });
|
||||
|
||||
const context = {};
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.metrics.errorFrequency.recent_errors).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Overall Pressure Level Calculation', () => {
|
||||
test('should calculate NORMAL when all metrics low', () => {
|
||||
const context = {
|
||||
token_usage: 0.1,
|
||||
conversation_length: 5,
|
||||
task_depth: 1,
|
||||
errors_recent: 0
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.level).toBe('NORMAL');
|
||||
expect(result.overall_score).toBeLessThan(0.3);
|
||||
});
|
||||
|
||||
test('should calculate CRITICAL when multiple metrics high', () => {
|
||||
const context = {
|
||||
token_usage: 0.8,
|
||||
conversation_length: 90,
|
||||
task_depth: 6,
|
||||
errors_recent: 8
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(['CRITICAL', 'DANGEROUS']).toContain(result.level);
|
||||
expect(result.overall_score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should weight token usage heavily in calculation', () => {
|
||||
const highToken = monitor.analyzePressure({ token_usage: 0.9 });
|
||||
const highErrors = monitor.analyzePressure({ errors_recent: 10 });
|
||||
|
||||
// High token usage should produce higher pressure than high errors alone
|
||||
expect(highToken.overall_score).toBeGreaterThan(highErrors.overall_score);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Pressure Level Thresholds', () => {
|
||||
test('should use correct thresholds for each level', () => {
|
||||
const levels = [
|
||||
{ score: 0.1, expected: 'NORMAL' },
|
||||
{ score: 0.35, expected: 'ELEVATED' },
|
||||
{ score: 0.55, expected: 'HIGH' },
|
||||
{ score: 0.75, expected: 'CRITICAL' },
|
||||
{ score: 0.95, expected: 'DANGEROUS' }
|
||||
];
|
||||
|
||||
levels.forEach(({ score, expected }) => {
|
||||
const result = monitor._determinePressureLevel(score);
|
||||
expect(result).toBe(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Recommendations', () => {
|
||||
test('should recommend normal operation at NORMAL pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.2,
|
||||
conversation_length: 10
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.recommendations).toContain('CONTINUE_NORMAL');
|
||||
});
|
||||
|
||||
test('should recommend increased verification at ELEVATED pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.45,
|
||||
conversation_length: 40
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.recommendations).toContain('INCREASE_VERIFICATION');
|
||||
});
|
||||
|
||||
test('should recommend context refresh at HIGH pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.65,
|
||||
conversation_length: 75
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.recommendations).toContain('SUGGEST_CONTEXT_REFRESH');
|
||||
});
|
||||
|
||||
test('should recommend mandatory verification at CRITICAL pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.8,
|
||||
errors_recent: 8
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.recommendations).toContain('MANDATORY_VERIFICATION');
|
||||
});
|
||||
|
||||
test('should recommend immediate halt at DANGEROUS pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.95,
|
||||
conversation_length: 120,
|
||||
errors_recent: 15
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.recommendations).toContain('IMMEDIATE_HALT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('27027 Incident Correlation', () => {
|
||||
test('should recognize 27027-like pressure conditions', () => {
|
||||
// Simulate conditions that led to 27027 failure
|
||||
const context = {
|
||||
token_usage: 0.535, // 107k/200k
|
||||
conversation_length: 50,
|
||||
task_depth: 3,
|
||||
errors_recent: 0,
|
||||
debugging_session: true
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.level).toMatch(/ELEVATED|HIGH/);
|
||||
expect(result.warnings).toContain('Conditions similar to documented failure modes');
|
||||
});
|
||||
|
||||
test('should flag pattern-reliance risk at high pressure', () => {
|
||||
const context = {
|
||||
token_usage: 0.6,
|
||||
conversation_length: 60
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.risks).toContain('increased pattern reliance');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Pressure History Tracking', () => {
|
||||
test('should track pressure over time', () => {
|
||||
monitor.analyzePressure({ token_usage: 0.2 });
|
||||
monitor.analyzePressure({ token_usage: 0.4 });
|
||||
monitor.analyzePressure({ token_usage: 0.6 });
|
||||
|
||||
const history = monitor.getPressureHistory();
|
||||
|
||||
expect(history.length).toBe(3);
|
||||
expect(history[0].level).toBe('NORMAL');
|
||||
expect(history[2].level).toMatch(/ELEVATED|HIGH/);
|
||||
});
|
||||
|
||||
test('should detect pressure escalation trends', () => {
|
||||
monitor.analyzePressure({ token_usage: 0.3 });
|
||||
monitor.analyzePressure({ token_usage: 0.5 });
|
||||
monitor.analyzePressure({ token_usage: 0.7 });
|
||||
|
||||
const result = monitor.analyzePressure({ token_usage: 0.8 });
|
||||
|
||||
expect(result.trend).toBe('escalating');
|
||||
expect(result.warnings).toContain('Pressure is escalating rapidly');
|
||||
});
|
||||
|
||||
test('should detect pressure de-escalation', () => {
|
||||
monitor.analyzePressure({ token_usage: 0.8 });
|
||||
monitor.analyzePressure({ token_usage: 0.6 });
|
||||
monitor.analyzePressure({ token_usage: 0.4 });
|
||||
|
||||
const result = monitor.analyzePressure({ token_usage: 0.3 });
|
||||
|
||||
expect(result.trend).toBe('improving');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Recording and Analysis', () => {
|
||||
test('should record errors with metadata', () => {
|
||||
monitor.recordError({
|
||||
type: 'platform_assumption',
|
||||
description: 'Used port 27017 instead of 27027',
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
const stats = monitor.getStats();
|
||||
|
||||
expect(stats.total_errors).toBe(1);
|
||||
expect(stats.error_types.platform_assumption).toBe(1);
|
||||
});
|
||||
|
||||
test('should detect error clustering', () => {
|
||||
// Record multiple errors in short time
|
||||
for (let i = 0; i < 5; i++) {
|
||||
monitor.recordError({ type: 'syntax_error' });
|
||||
}
|
||||
|
||||
const context = {};
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
expect(result.warnings).toContain('Error clustering detected');
|
||||
});
|
||||
|
||||
test('should track error patterns by type', () => {
|
||||
monitor.recordError({ type: 'platform_assumption' });
|
||||
monitor.recordError({ type: 'platform_assumption' });
|
||||
monitor.recordError({ type: 'context_loss' });
|
||||
|
||||
const stats = monitor.getStats();
|
||||
|
||||
expect(stats.error_types.platform_assumption).toBe(2);
|
||||
expect(stats.error_types.context_loss).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Reset and Cleanup', () => {
|
||||
test('should reset pressure monitoring state', () => {
|
||||
monitor.analyzePressure({ token_usage: 0.8 });
|
||||
monitor.recordError({ type: 'test' });
|
||||
|
||||
monitor.reset();
|
||||
|
||||
const stats = monitor.getStats();
|
||||
const history = monitor.getPressureHistory();
|
||||
|
||||
expect(stats.total_analyses).toBe(0);
|
||||
expect(history).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('should clear error history on reset', () => {
|
||||
monitor.recordError({ type: 'test1' });
|
||||
monitor.recordError({ type: 'test2' });
|
||||
|
||||
monitor.reset();
|
||||
|
||||
const stats = monitor.getStats();
|
||||
expect(stats.total_errors).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Singleton Pattern', () => {
|
||||
test('should export singleton instance with required methods', () => {
|
||||
expect(typeof monitor.analyzePressure).toBe('function');
|
||||
expect(typeof monitor.recordError).toBe('function');
|
||||
expect(typeof monitor.getStats).toBe('function');
|
||||
});
|
||||
|
||||
test('should maintain pressure history across calls', () => {
|
||||
if (monitor.analyzePressure && monitor.getPressureHistory) {
|
||||
monitor.analyzePressure({ token_usage: 0.5 });
|
||||
|
||||
const history = monitor.getPressureHistory();
|
||||
|
||||
expect(history).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Statistics Tracking', () => {
|
||||
test('should track analysis statistics', () => {
|
||||
const stats = monitor.getStats();
|
||||
|
||||
expect(stats).toHaveProperty('total_analyses');
|
||||
expect(stats).toHaveProperty('by_level');
|
||||
expect(stats).toHaveProperty('total_errors');
|
||||
});
|
||||
|
||||
test('should increment analysis count after analyzePressure()', () => {
|
||||
const before = monitor.getStats().total_analyses;
|
||||
|
||||
monitor.analyzePressure({ token_usage: 0.3 });
|
||||
|
||||
const after = monitor.getStats().total_analyses;
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
test('should track pressure level distribution', () => {
|
||||
monitor.analyzePressure({ token_usage: 0.2 }); // NORMAL
|
||||
monitor.analyzePressure({ token_usage: 0.4 }); // ELEVATED
|
||||
monitor.analyzePressure({ token_usage: 0.6 }); // HIGH
|
||||
|
||||
const stats = monitor.getStats();
|
||||
|
||||
expect(stats.by_level.NORMAL).toBeGreaterThan(0);
|
||||
expect(stats.by_level.ELEVATED).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
test('should handle empty context gracefully', () => {
|
||||
const result = monitor.analyzePressure({});
|
||||
|
||||
expect(result.level).toBe('NORMAL');
|
||||
expect(result.overall_score).toBeDefined();
|
||||
});
|
||||
|
||||
test('should handle null context gracefully', () => {
|
||||
expect(() => {
|
||||
monitor.analyzePressure(null);
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
test('should handle invalid token_usage values', () => {
|
||||
const result = monitor.analyzePressure({ token_usage: -1 });
|
||||
|
||||
expect(result.metrics.tokenUsage.score).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('should handle token_usage over 1.0', () => {
|
||||
const result = monitor.analyzePressure({ token_usage: 1.5 });
|
||||
|
||||
expect(result.level).toBe('DANGEROUS');
|
||||
expect(result.recommendations).toContain('IMMEDIATE_HALT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Contextual Adjustments', () => {
|
||||
test('should consider debugging context in pressure calculation', () => {
|
||||
const normalContext = { token_usage: 0.5 };
|
||||
const debugContext = { token_usage: 0.5, debugging_session: true };
|
||||
|
||||
const normalResult = monitor.analyzePressure(normalContext);
|
||||
const debugResult = monitor.analyzePressure(debugContext);
|
||||
|
||||
// Debugging increases pressure
|
||||
expect(debugResult.overall_score).toBeGreaterThanOrEqual(normalResult.overall_score);
|
||||
});
|
||||
|
||||
test('should adjust for production environment', () => {
|
||||
const context = {
|
||||
token_usage: 0.6,
|
||||
environment: 'production'
|
||||
};
|
||||
|
||||
const result = monitor.analyzePressure(context);
|
||||
|
||||
// Production should lower threshold for warnings
|
||||
expect(result.warnings.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Warning and Alert Generation', () => {
|
||||
test('should generate appropriate warnings for each pressure level', () => {
|
||||
const dangerous = monitor.analyzePressure({ token_usage: 0.95 });
|
||||
|
||||
expect(dangerous.warnings.length).toBeGreaterThan(0);
|
||||
expect(dangerous.warnings.some(w => w.includes('critical'))).toBe(true);
|
||||
});
|
||||
|
||||
test('should include specific metrics in warnings', () => {
|
||||
const result = monitor.analyzePressure({
|
||||
token_usage: 0.8,
|
||||
errors_recent: 10
|
||||
});
|
||||
|
||||
expect(result.warnings.some(w => w.includes('token'))).toBe(true);
|
||||
expect(result.warnings.some(w => w.includes('error'))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
589
tests/unit/CrossReferenceValidator.test.js
Normal file
589
tests/unit/CrossReferenceValidator.test.js
Normal file
|
|
@ -0,0 +1,589 @@
|
|||
/**
|
||||
* Unit Tests for CrossReferenceValidator
|
||||
* Tests cross-reference validation to prevent pattern override of explicit instructions
|
||||
*/
|
||||
|
||||
const validator = require('../../src/services/CrossReferenceValidator.service');
|
||||
const classifier = require('../../src/services/InstructionPersistenceClassifier.service');
|
||||
|
||||
describe('CrossReferenceValidator', () => {
|
||||
beforeEach(() => {
|
||||
// Clear instruction history before each test
|
||||
if (validator.clearInstructions) {
|
||||
validator.clearInstructions();
|
||||
}
|
||||
});
|
||||
|
||||
describe('27027 Failure Mode Prevention', () => {
|
||||
test('should detect conflict when action uses default port instead of explicit instruction', () => {
|
||||
// Simulate explicit user instruction
|
||||
const instruction = classifier.classify({
|
||||
text: 'check MongoDB on port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
// Simulate AI action using wrong port
|
||||
const action = {
|
||||
type: 'database_query',
|
||||
description: 'Connect to MongoDB',
|
||||
parameters: {
|
||||
port: 27017,
|
||||
database: 'family_history'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts).toHaveLength(1);
|
||||
expect(result.conflicts[0].parameter).toBe('port');
|
||||
expect(result.conflicts[0].severity).toBe('CRITICAL');
|
||||
});
|
||||
|
||||
test('should approve action when port matches explicit instruction', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'check MongoDB on port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'database_query',
|
||||
description: 'Connect to MongoDB',
|
||||
parameters: {
|
||||
port: 27027,
|
||||
database: 'family_history'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
expect(result.conflicts).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Conflict Detection', () => {
|
||||
test('should detect parameter conflicts between action and instruction', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'use React for the frontend, not Vue',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'install_package',
|
||||
description: 'Install Vue.js framework',
|
||||
parameters: {
|
||||
package: 'vue',
|
||||
framework: 'vue'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should not flag conflicts for unrelated actions', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'use MongoDB on port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'file_write',
|
||||
description: 'Create package.json',
|
||||
parameters: {
|
||||
file: 'package.json',
|
||||
content: '{}'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
expect(result.conflicts).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Relevance Calculation', () => {
|
||||
test('should prioritize recent instructions over older ones', () => {
|
||||
const oldInstruction = classifier.classify({
|
||||
text: 'use port 27017',
|
||||
context: {},
|
||||
source: 'user',
|
||||
timestamp: new Date(Date.now() - 3600000) // 1 hour ago
|
||||
});
|
||||
|
||||
const recentInstruction = classifier.classify({
|
||||
text: 'actually, use port 27027 instead',
|
||||
context: {},
|
||||
source: 'user',
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
validator.addInstruction(oldInstruction);
|
||||
validator.addInstruction(recentInstruction);
|
||||
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
description: 'Connect to database',
|
||||
parameters: {
|
||||
port: 27017
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, {
|
||||
recent_instructions: [oldInstruction, recentInstruction]
|
||||
});
|
||||
|
||||
// Should conflict with recent instruction, not old one
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts[0].instruction.text).toContain('27027');
|
||||
});
|
||||
|
||||
test('should prioritize high-persistence instructions', () => {
|
||||
const lowPersistence = classifier.classify({
|
||||
text: 'try using port 8000',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
const highPersistence = classifier.classify({
|
||||
text: 'you must use port 9000',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(lowPersistence);
|
||||
validator.addInstruction(highPersistence);
|
||||
|
||||
const action = {
|
||||
type: 'start_server',
|
||||
description: 'Start development server',
|
||||
parameters: {
|
||||
port: 8000
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, {
|
||||
recent_instructions: [lowPersistence, highPersistence]
|
||||
});
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts[0].instruction.text).toContain('must');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Conflict Severity Levels', () => {
|
||||
test('should assign CRITICAL severity to conflicts with HIGH persistence explicit instructions', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'never use HTTP, always use HTTPS',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'api_request',
|
||||
description: 'Fetch data',
|
||||
parameters: {
|
||||
protocol: 'http',
|
||||
url: 'http://example.com'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts[0].severity).toBe('CRITICAL');
|
||||
});
|
||||
|
||||
test('should assign WARNING severity to conflicts with MEDIUM persistence instructions', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'prefer using async/await over callbacks',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'code_generation',
|
||||
description: 'Generate function with callbacks',
|
||||
parameters: {
|
||||
pattern: 'callback'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(['WARNING', 'APPROVED']).toContain(result.status);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Lookback Window', () => {
|
||||
test('should only consider instructions within lookback window', () => {
|
||||
// Create 60 instructions
|
||||
for (let i = 0; i < 60; i++) {
|
||||
const instruction = classifier.classify({
|
||||
text: `instruction ${i}`,
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
validator.addInstruction(instruction);
|
||||
}
|
||||
|
||||
// Most recent instruction says use port 27027
|
||||
const recentInstruction = classifier.classify({
|
||||
text: 'use port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
validator.addInstruction(recentInstruction);
|
||||
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: {
|
||||
port: 27017
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, {
|
||||
recent_instructions: validator.getRecentInstructions()
|
||||
});
|
||||
|
||||
// Should detect conflict with recent instruction
|
||||
expect(result.status).toBe('REJECTED');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Parameter Extraction', () => {
|
||||
test('should extract port numbers from action parameters', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: {
|
||||
host: 'localhost',
|
||||
port: 27017,
|
||||
database: 'test'
|
||||
}
|
||||
};
|
||||
|
||||
const extracted = validator._extractActionParameters(action);
|
||||
|
||||
expect(extracted).toHaveProperty('port', 27017);
|
||||
});
|
||||
|
||||
test('should extract port numbers from action description', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
description: 'Connect to MongoDB on port 27017'
|
||||
};
|
||||
|
||||
const extracted = validator._extractActionParameters(action);
|
||||
|
||||
expect(extracted).toHaveProperty('port', '27017');
|
||||
});
|
||||
|
||||
test('should extract database names from parameters', () => {
|
||||
const action = {
|
||||
type: 'query',
|
||||
parameters: {
|
||||
database: 'family_history',
|
||||
collection: 'users'
|
||||
}
|
||||
};
|
||||
|
||||
const extracted = validator._extractActionParameters(action);
|
||||
|
||||
expect(extracted).toHaveProperty('database', 'family_history');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Instruction Management', () => {
|
||||
test('should add instructions to history', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'test instruction',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const recent = validator.getRecentInstructions();
|
||||
expect(recent).toContainEqual(instruction);
|
||||
});
|
||||
|
||||
test('should clear instruction history', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'test instruction',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
expect(validator.getRecentInstructions().length).toBeGreaterThan(0);
|
||||
|
||||
validator.clearInstructions();
|
||||
expect(validator.getRecentInstructions()).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('should maintain instruction order (most recent first)', () => {
|
||||
const first = classifier.classify({
|
||||
text: 'first',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
const second = classifier.classify({
|
||||
text: 'second',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(first);
|
||||
validator.addInstruction(second);
|
||||
|
||||
const recent = validator.getRecentInstructions();
|
||||
expect(recent[0].text).toBe('second');
|
||||
expect(recent[1].text).toBe('first');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Complex Scenarios', () => {
|
||||
test('should handle multiple conflicting parameters', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'connect to database family_history on host localhost port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: {
|
||||
host: 'remotehost',
|
||||
port: 27017,
|
||||
database: 'other_db'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.conflicts.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
test('should handle partial matches correctly', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'use port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
// Action that matches one parameter but not all
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: {
|
||||
port: 27027,
|
||||
ssl: false // Not mentioned in instruction
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Validation Decision Logic', () => {
|
||||
test('should reject when critical conflicts exist', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'never delete user data without confirmation',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'delete_users',
|
||||
description: 'Delete all inactive users',
|
||||
parameters: {
|
||||
confirmed: false
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('REJECTED');
|
||||
expect(result.required_action).toBe('REQUEST_CLARIFICATION');
|
||||
});
|
||||
|
||||
test('should warn when minor conflicts exist', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'consider using ESM imports instead of CommonJS',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'code_generation',
|
||||
description: 'Generate CommonJS module',
|
||||
parameters: {
|
||||
module_type: 'commonjs'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(['WARNING', 'APPROVED']).toContain(result.status);
|
||||
});
|
||||
|
||||
test('should approve when no conflicts exist', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'use TypeScript for new files',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
const action = {
|
||||
type: 'create_file',
|
||||
description: 'Create new TypeScript component',
|
||||
parameters: {
|
||||
extension: '.ts'
|
||||
}
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
test('should handle validation with empty instruction history', () => {
|
||||
const action = {
|
||||
type: 'test',
|
||||
parameters: { port: 27017 }
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
expect(result.conflicts).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('should handle action with no parameters', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'test',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
const action = {
|
||||
type: 'simple_action'
|
||||
};
|
||||
|
||||
const result = validator.validate(action, { recent_instructions: [instruction] });
|
||||
|
||||
expect(result.status).toBe('APPROVED');
|
||||
});
|
||||
|
||||
test('should handle null or undefined action gracefully', () => {
|
||||
expect(() => {
|
||||
validator.validate(null, { recent_instructions: [] });
|
||||
}).not.toThrow();
|
||||
|
||||
expect(() => {
|
||||
validator.validate(undefined, { recent_instructions: [] });
|
||||
}).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Singleton Pattern', () => {
|
||||
test('should export singleton instance with required methods', () => {
|
||||
expect(typeof validator.validate).toBe('function');
|
||||
expect(typeof validator.addInstruction).toBe('function');
|
||||
expect(typeof validator.getStats).toBe('function');
|
||||
});
|
||||
|
||||
test('should maintain instruction history across calls', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'test',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
if (validator.addInstruction) {
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
if (validator.getRecentInstructions) {
|
||||
const history = validator.getRecentInstructions();
|
||||
expect(history).toBeDefined();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Statistics Tracking', () => {
|
||||
test('should track validation statistics', () => {
|
||||
const stats = validator.getStats();
|
||||
|
||||
expect(stats).toHaveProperty('total_validations');
|
||||
expect(stats).toHaveProperty('conflicts_detected');
|
||||
expect(stats).toHaveProperty('rejections');
|
||||
});
|
||||
|
||||
test('should increment validation count after validate()', () => {
|
||||
const before = validator.getStats().total_validations;
|
||||
|
||||
validator.validate(
|
||||
{ type: 'test', parameters: {} },
|
||||
{ recent_instructions: [] }
|
||||
);
|
||||
|
||||
const after = validator.getStats().total_validations;
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
test('should track conflict detection rate', () => {
|
||||
const instruction = classifier.classify({
|
||||
text: 'use port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
validator.addInstruction(instruction);
|
||||
|
||||
// Action with conflict
|
||||
validator.validate(
|
||||
{ type: 'connect', parameters: { port: 27017 } },
|
||||
{ recent_instructions: [instruction] }
|
||||
);
|
||||
|
||||
const stats = validator.getStats();
|
||||
expect(stats.conflicts_detected).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
411
tests/unit/InstructionPersistenceClassifier.test.js
Normal file
411
tests/unit/InstructionPersistenceClassifier.test.js
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
/**
|
||||
* Unit Tests for InstructionPersistenceClassifier
|
||||
* Tests quadrant classification, persistence calculation, and verification requirements
|
||||
*/
|
||||
|
||||
const classifier = require('../../src/services/InstructionPersistenceClassifier.service');
|
||||
|
||||
describe('InstructionPersistenceClassifier', () => {
|
||||
// Classifier is a singleton instance, no setup needed
|
||||
beforeEach(() => {
|
||||
// Could reset stats here if needed
|
||||
});
|
||||
|
||||
describe('Quadrant Classification', () => {
|
||||
test('should classify strategic values statements as STRATEGIC', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Always prioritize user privacy over convenience',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('STRATEGIC');
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
});
|
||||
|
||||
test('should classify explicit port specification as TACTICAL with HIGH persistence', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'check port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('TACTICAL');
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
expect(result.verification_required).toBe('MANDATORY');
|
||||
});
|
||||
|
||||
test('should classify technical code fixes as SYSTEM', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'fix the syntax error in line 42',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('SYSTEM');
|
||||
expect(result.persistence).toBe('MEDIUM');
|
||||
});
|
||||
|
||||
test('should classify operational process instructions as OPERATIONAL', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'for this project, always use React hooks instead of class components',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('OPERATIONAL');
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
});
|
||||
|
||||
test('should classify exploratory requests as STOCHASTIC', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'explore different approaches to implementing user authentication',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('STOCHASTIC');
|
||||
expect(result.persistence).toBe('MEDIUM');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Persistence Level Calculation', () => {
|
||||
test('should assign HIGH persistence to explicit instructions with must/never', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'you must never commit credentials to the repository',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
expect(result.explicitness).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should assign MEDIUM persistence to general guidelines', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'try to keep functions under 50 lines',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.persistence).toBe('MEDIUM');
|
||||
});
|
||||
|
||||
test('should assign LOW persistence to one-time immediate actions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'print the current directory',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.persistence).toBe('LOW');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Verification Requirements', () => {
|
||||
test('should require MANDATORY verification for HIGH persistence STRATEGIC instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Never deploy to production without human approval',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.verification_required).toBe('MANDATORY');
|
||||
});
|
||||
|
||||
test('should require RECOMMENDED verification for MEDIUM persistence instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'prefer async/await over callbacks',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(['RECOMMENDED', 'MANDATORY']).toContain(result.verification_required);
|
||||
});
|
||||
|
||||
test('should allow OPTIONAL verification for LOW persistence instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'show me the package.json file',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(['OPTIONAL', 'RECOMMENDED']).toContain(result.verification_required);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Temporal Scope Detection', () => {
|
||||
test('should detect PERMANENT scope for always/never statements', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Always validate user input before database queries',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata.temporal_scope).toBe('PERMANENT');
|
||||
});
|
||||
|
||||
test('should detect PROJECT scope for project-specific instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'For this project, use MongoDB on port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata.temporal_scope).toBe('PROJECT');
|
||||
});
|
||||
|
||||
test('should detect IMMEDIATE scope for right now statements', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'right now, restart the development server',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata.temporal_scope).toBe('IMMEDIATE');
|
||||
});
|
||||
|
||||
test('should detect SESSION scope for context-specific instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'for the rest of this conversation, use verbose logging',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata.temporal_scope).toBe('SESSION');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Explicitness Measurement', () => {
|
||||
test('should score high explicitness for instructions with explicit markers', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'You must specifically use port 27027, not the default port',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.explicitness).toBeGreaterThan(0.8);
|
||||
});
|
||||
|
||||
test('should score low explicitness for implicit suggestions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'maybe consider using port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.explicitness).toBeLessThan(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('27027 Failure Mode Prevention', () => {
|
||||
test('should classify port specification with HIGH persistence and MANDATORY verification', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'check MongoDB on port 27027 for the family-history collection',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('TACTICAL');
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
expect(result.verification_required).toBe('MANDATORY');
|
||||
expect(result.metadata.extracted_parameters).toHaveProperty('port', '27027');
|
||||
});
|
||||
|
||||
test('should extract and preserve specific parameter values', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'connect to database family_history on port 27027',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata.extracted_parameters).toMatchObject({
|
||||
port: '27027',
|
||||
database: 'family_history'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Metadata Preservation', () => {
|
||||
test('should preserve timestamp', () => {
|
||||
const before = new Date();
|
||||
const result = classifier.classify({
|
||||
text: 'test instruction',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
const after = new Date();
|
||||
|
||||
expect(result.timestamp).toBeInstanceOf(Date);
|
||||
expect(result.timestamp.getTime()).toBeGreaterThanOrEqual(before.getTime());
|
||||
expect(result.timestamp.getTime()).toBeLessThanOrEqual(after.getTime());
|
||||
});
|
||||
|
||||
test('should preserve source attribution', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'test instruction',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.source).toBe('user');
|
||||
});
|
||||
|
||||
test('should include metadata object with all required fields', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Always use TypeScript for new projects',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.metadata).toHaveProperty('temporal_scope');
|
||||
expect(result.metadata).toHaveProperty('extracted_parameters');
|
||||
expect(result.metadata).toHaveProperty('context_snapshot');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
test('should handle empty text gracefully', () => {
|
||||
const result = classifier.classify({
|
||||
text: '',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('STOCHASTIC');
|
||||
expect(result.persistence).toBe('LOW');
|
||||
});
|
||||
|
||||
test('should handle very long instructions', () => {
|
||||
const longText = 'Always ' + 'do this '.repeat(100) + 'when implementing features';
|
||||
const result = classifier.classify({
|
||||
text: longText,
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBeDefined();
|
||||
expect(result.persistence).toBeDefined();
|
||||
});
|
||||
|
||||
test('should handle special characters and unicode', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Use emojis 🔒 for security-related messages',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBeDefined();
|
||||
expect(result.text).toContain('🔒');
|
||||
});
|
||||
|
||||
test('should handle code blocks in instructions', () => {
|
||||
const result = classifier.classify({
|
||||
text: 'Use this pattern: const foo = async () => { await bar(); }',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBeDefined();
|
||||
expect(result.metadata.extracted_parameters).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Classification Consistency', () => {
|
||||
test('should produce consistent results for identical inputs', () => {
|
||||
const text = 'Always validate user input before database operations';
|
||||
const result1 = classifier.classify({ text, context: {}, source: 'user' });
|
||||
const result2 = classifier.classify({ text, context: {}, source: 'user' });
|
||||
|
||||
expect(result1.quadrant).toBe(result2.quadrant);
|
||||
expect(result1.persistence).toBe(result2.persistence);
|
||||
expect(result1.explicitness).toBe(result2.explicitness);
|
||||
});
|
||||
|
||||
test('should handle variations in capitalization consistently', () => {
|
||||
const lower = classifier.classify({ text: 'always use https', context: {}, source: 'user' });
|
||||
const upper = classifier.classify({ text: 'ALWAYS USE HTTPS', context: {}, source: 'user' });
|
||||
|
||||
expect(lower.quadrant).toBe(upper.quadrant);
|
||||
expect(lower.persistence).toBe(upper.persistence);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Context Integration', () => {
|
||||
test('should consider conversation context in classification', () => {
|
||||
const context = {
|
||||
recent_topics: ['security', 'authentication'],
|
||||
pressure_level: 'NORMAL'
|
||||
};
|
||||
|
||||
const result = classifier.classify({
|
||||
text: 'never store passwords in plain text',
|
||||
context,
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
expect(result.quadrant).toBe('STRATEGIC');
|
||||
expect(result.persistence).toBe('HIGH');
|
||||
});
|
||||
|
||||
test('should adjust verification requirements based on context pressure', () => {
|
||||
const highPressureContext = {
|
||||
token_usage: 0.9,
|
||||
errors_recent: 5,
|
||||
conversation_length: 100
|
||||
};
|
||||
|
||||
const result = classifier.classify({
|
||||
text: 'update the database schema',
|
||||
context: highPressureContext,
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
// High pressure should increase verification requirements
|
||||
expect(['RECOMMENDED', 'MANDATORY']).toContain(result.verification_required);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Singleton Pattern', () => {
|
||||
test('should export singleton instance', () => {
|
||||
// Verify the exported object has the expected methods
|
||||
expect(typeof classifier.classify).toBe('function');
|
||||
expect(typeof classifier.getStats).toBe('function');
|
||||
});
|
||||
|
||||
test('should maintain classification count across calls', () => {
|
||||
const initialCount = classifier.getStats().total_classifications;
|
||||
|
||||
classifier.classify({ text: 'test', context: {}, source: 'user' });
|
||||
|
||||
const newCount = classifier.getStats().total_classifications;
|
||||
|
||||
expect(newCount).toBe(initialCount + 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Statistics Tracking', () => {
|
||||
test('should track classification statistics', () => {
|
||||
const stats = classifier.getStats();
|
||||
|
||||
expect(stats).toHaveProperty('total_classifications');
|
||||
expect(stats).toHaveProperty('by_quadrant');
|
||||
expect(stats).toHaveProperty('by_persistence');
|
||||
expect(stats).toHaveProperty('by_verification');
|
||||
});
|
||||
|
||||
test('should increment classification count after classify()', () => {
|
||||
const before = classifier.getStats().total_classifications;
|
||||
|
||||
classifier.classify({
|
||||
text: 'test instruction',
|
||||
context: {},
|
||||
source: 'user'
|
||||
});
|
||||
|
||||
const after = classifier.getStats().total_classifications;
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
});
|
||||
});
|
||||
672
tests/unit/MetacognitiveVerifier.test.js
Normal file
672
tests/unit/MetacognitiveVerifier.test.js
Normal file
|
|
@ -0,0 +1,672 @@
|
|||
/**
|
||||
* Unit Tests for MetacognitiveVerifier
|
||||
* Tests metacognitive self-verification before action execution
|
||||
*/
|
||||
|
||||
const verifier = require('../../src/services/MetacognitiveVerifier.service');
|
||||
|
||||
describe('MetacognitiveVerifier', () => {
|
||||
beforeEach(() => {
|
||||
// Verifier is a singleton instance
|
||||
});
|
||||
|
||||
describe('Alignment Verification', () => {
|
||||
test('should verify action aligns with stated reasoning', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: { port: 27027 }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'User explicitly requested port 27027',
|
||||
evidence: ['user instruction: use port 27027']
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.alignment.passed).toBe(true);
|
||||
expect(result.checks.alignment.score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should detect misalignment between action and reasoning', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: { port: 27017 }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'User explicitly requested port 27027',
|
||||
evidence: ['user instruction: use port 27027']
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.alignment.passed).toBe(false);
|
||||
expect(result.checks.alignment.issues).toContain('action parameters conflict with reasoning');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Coherence Verification', () => {
|
||||
test('should verify reasoning is internally consistent', () => {
|
||||
const action = {
|
||||
type: 'install_package',
|
||||
parameters: { package: 'react' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Installing React as requested',
|
||||
steps: [
|
||||
'User asked for React installation',
|
||||
'React is the appropriate package',
|
||||
'Install React via npm'
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.coherence.passed).toBe(true);
|
||||
expect(result.checks.coherence.score).toBeGreaterThan(0.6);
|
||||
});
|
||||
|
||||
test('should detect logical contradictions in reasoning', () => {
|
||||
const action = {
|
||||
type: 'install_package',
|
||||
parameters: { package: 'vue' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Installing React framework',
|
||||
steps: [
|
||||
'Install React',
|
||||
'Actually using Vue',
|
||||
'Run Vue installation'
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.coherence.passed).toBe(false);
|
||||
expect(result.checks.coherence.issues).toContain('reasoning contains contradictions');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Completeness Verification', () => {
|
||||
test('should verify all necessary steps are included', () => {
|
||||
const action = {
|
||||
type: 'deploy',
|
||||
parameters: { environment: 'production' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Deploy to production',
|
||||
steps: [
|
||||
'Run tests',
|
||||
'Build production bundle',
|
||||
'Backup current version',
|
||||
'Deploy new version',
|
||||
'Verify deployment'
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.completeness.passed).toBe(true);
|
||||
expect(result.checks.completeness.score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should detect missing critical steps', () => {
|
||||
const action = {
|
||||
type: 'deploy',
|
||||
parameters: { environment: 'production' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Deploy to production',
|
||||
steps: [
|
||||
'Deploy new version'
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.completeness.passed).toBe(false);
|
||||
expect(result.checks.completeness.missing_considerations).toContain('testing');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Safety Verification', () => {
|
||||
test('should verify safe operations pass safety check', () => {
|
||||
const action = {
|
||||
type: 'read_file',
|
||||
parameters: { file: 'config.json' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Reading configuration file to check settings'
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.safety.passed).toBe(true);
|
||||
expect(result.checks.safety.risk_level).toBe('LOW');
|
||||
});
|
||||
|
||||
test('should flag dangerous operations', () => {
|
||||
const action = {
|
||||
type: 'delete_all',
|
||||
parameters: { table: 'users' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Cleaning up user table'
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.safety.passed).toBe(false);
|
||||
expect(result.checks.safety.risk_level).toBe('HIGH');
|
||||
expect(result.checks.safety.concerns).toContain('destructive operation');
|
||||
});
|
||||
|
||||
test('should require explicit confirmation for risky actions', () => {
|
||||
const action = {
|
||||
type: 'modify_schema',
|
||||
parameters: { table: 'users' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Update database schema'
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.requires_confirmation).toBe(true);
|
||||
expect(result.checks.safety.risk_level).toMatch(/MEDIUM|HIGH/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Alternative Consideration', () => {
|
||||
test('should verify alternatives were considered', () => {
|
||||
const action = {
|
||||
type: 'implementation',
|
||||
parameters: { approach: 'A' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Using approach A',
|
||||
alternatives_considered: [
|
||||
'Approach A: Fast but uses more memory',
|
||||
'Approach B: Slower but memory efficient',
|
||||
'Selected A for performance priority'
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.alternatives.passed).toBe(true);
|
||||
expect(result.checks.alternatives.score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should flag lack of alternative consideration', () => {
|
||||
const action = {
|
||||
type: 'implementation',
|
||||
parameters: { approach: 'A' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Using approach A',
|
||||
alternatives_considered: []
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.checks.alternatives.passed).toBe(false);
|
||||
expect(result.checks.alternatives.issues).toContain('no alternatives considered');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Overall Confidence Calculation', () => {
|
||||
test('should calculate high confidence when all checks pass', () => {
|
||||
const action = {
|
||||
type: 'safe_operation',
|
||||
parameters: { file: 'test.txt' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Safe file read operation',
|
||||
evidence: ['user requested', 'file exists', 'read-only'],
|
||||
steps: ['locate file', 'read contents', 'return data'],
|
||||
alternatives_considered: ['direct read', 'streamed read']
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.confidence).toBeGreaterThan(0.7);
|
||||
expect(result.decision).toBe('PROCEED');
|
||||
});
|
||||
|
||||
test('should calculate low confidence when checks fail', () => {
|
||||
const action = {
|
||||
type: 'risky_operation',
|
||||
parameters: { destructive: true }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Maybe do this',
|
||||
evidence: [],
|
||||
steps: ['do it']
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.confidence).toBeLessThan(0.5);
|
||||
expect(result.decision).toMatch(/BLOCK|REQUEST_CLARIFICATION/);
|
||||
});
|
||||
|
||||
test('should weight checks appropriately', () => {
|
||||
// Safety failures should heavily impact confidence
|
||||
const unsafeAction = {
|
||||
type: 'delete_database'
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Delete database',
|
||||
evidence: ['complete analysis'],
|
||||
steps: ['backup', 'delete', 'verify'],
|
||||
alternatives_considered: ['archive instead']
|
||||
};
|
||||
|
||||
const result = verifier.verify(unsafeAction, reasoning, {});
|
||||
|
||||
expect(result.confidence).toBeLessThan(0.6);
|
||||
expect(result.checks.safety.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Pressure-Adjusted Verification', () => {
|
||||
test('should increase verification strictness under high pressure', () => {
|
||||
const action = {
|
||||
type: 'database_update',
|
||||
parameters: { table: 'users' }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Update users table'
|
||||
};
|
||||
|
||||
const lowPressure = { pressure_level: 'NORMAL' };
|
||||
const highPressure = { pressure_level: 'CRITICAL', token_usage: 0.9 };
|
||||
|
||||
const lowResult = verifier.verify(action, reasoning, lowPressure);
|
||||
const highResult = verifier.verify(action, reasoning, highPressure);
|
||||
|
||||
// High pressure should reduce confidence
|
||||
expect(highResult.confidence).toBeLessThan(lowResult.confidence);
|
||||
expect(highResult.pressure_adjustment).toBeLessThan(1.0);
|
||||
});
|
||||
|
||||
test('should require higher confidence threshold under pressure', () => {
|
||||
const action = {
|
||||
type: 'moderate_risk',
|
||||
parameters: {}
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Moderate risk operation',
|
||||
evidence: ['some evidence']
|
||||
};
|
||||
|
||||
const criticalPressure = {
|
||||
pressure_level: 'CRITICAL',
|
||||
errors_recent: 10
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, criticalPressure);
|
||||
|
||||
expect(result.threshold_adjusted).toBe(true);
|
||||
expect(result.required_confidence).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should block operations at DANGEROUS pressure', () => {
|
||||
const action = {
|
||||
type: 'any_operation'
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Well-reasoned action'
|
||||
};
|
||||
|
||||
const dangerousPressure = {
|
||||
pressure_level: 'DANGEROUS',
|
||||
token_usage: 0.95
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, dangerousPressure);
|
||||
|
||||
expect(result.decision).toBe('BLOCK');
|
||||
expect(result.reason).toContain('pressure too high');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Verification Decisions', () => {
|
||||
test('should return PROCEED for high confidence actions', () => {
|
||||
const result = verifier._makeDecision(0.85, {});
|
||||
|
||||
expect(result.decision).toBe('PROCEED');
|
||||
expect(result.requires_confirmation).toBe(false);
|
||||
});
|
||||
|
||||
test('should return REQUEST_CONFIRMATION for medium confidence', () => {
|
||||
const result = verifier._makeDecision(0.65, {});
|
||||
|
||||
expect(result.decision).toBe('REQUEST_CONFIRMATION');
|
||||
expect(result.requires_confirmation).toBe(true);
|
||||
});
|
||||
|
||||
test('should return REQUEST_CLARIFICATION for low confidence', () => {
|
||||
const result = verifier._makeDecision(0.45, {});
|
||||
|
||||
expect(result.decision).toBe('REQUEST_CLARIFICATION');
|
||||
});
|
||||
|
||||
test('should return BLOCK for very low confidence', () => {
|
||||
const result = verifier._makeDecision(0.2, {});
|
||||
|
||||
expect(result.decision).toBe('BLOCK');
|
||||
});
|
||||
});
|
||||
|
||||
describe('27027 Failure Mode Prevention', () => {
|
||||
test('should detect when action conflicts with explicit instruction', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: { port: 27017 }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Connecting to MongoDB on default port',
|
||||
evidence: ['MongoDB default is 27017']
|
||||
};
|
||||
|
||||
const context = {
|
||||
explicit_instructions: [
|
||||
{ text: 'use port 27027', timestamp: new Date() }
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, context);
|
||||
|
||||
expect(result.checks.alignment.passed).toBe(false);
|
||||
expect(result.decision).toMatch(/BLOCK|REQUEST_CLARIFICATION/);
|
||||
});
|
||||
|
||||
test('should approve when action matches explicit instruction', () => {
|
||||
const action = {
|
||||
type: 'database_connect',
|
||||
parameters: { port: 27027 }
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Connecting to MongoDB on port 27027 as instructed',
|
||||
evidence: ['User explicitly said port 27027']
|
||||
};
|
||||
|
||||
const context = {
|
||||
explicit_instructions: [
|
||||
{ text: 'use port 27027', timestamp: new Date() }
|
||||
]
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, context);
|
||||
|
||||
expect(result.checks.alignment.passed).toBe(true);
|
||||
expect(result.confidence).toBeGreaterThan(0.7);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Evidence Quality Assessment', () => {
|
||||
test('should assess evidence quality', () => {
|
||||
const reasoning = {
|
||||
explanation: 'Action is needed',
|
||||
evidence: [
|
||||
'User explicitly requested this',
|
||||
'Documentation confirms approach',
|
||||
'Tests validate correctness'
|
||||
]
|
||||
};
|
||||
|
||||
const quality = verifier._assessEvidenceQuality(reasoning);
|
||||
|
||||
expect(quality).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
test('should penalize weak evidence', () => {
|
||||
const reasoning = {
|
||||
explanation: 'Action is needed',
|
||||
evidence: [
|
||||
'I think this is right',
|
||||
'Maybe this works'
|
||||
]
|
||||
};
|
||||
|
||||
const quality = verifier._assessEvidenceQuality(reasoning);
|
||||
|
||||
expect(quality).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
test('should penalize missing evidence', () => {
|
||||
const reasoning = {
|
||||
explanation: 'Action is needed',
|
||||
evidence: []
|
||||
};
|
||||
|
||||
const quality = verifier._assessEvidenceQuality(reasoning);
|
||||
|
||||
expect(quality).toBeLessThan(0.3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
test('should handle null action gracefully', () => {
|
||||
expect(() => {
|
||||
verifier.verify(null, { explanation: 'test' }, {});
|
||||
}).not.toThrow();
|
||||
|
||||
const result = verifier.verify(null, { explanation: 'test' }, {});
|
||||
expect(result.decision).toBe('BLOCK');
|
||||
});
|
||||
|
||||
test('should handle null reasoning gracefully', () => {
|
||||
expect(() => {
|
||||
verifier.verify({ type: 'test' }, null, {});
|
||||
}).not.toThrow();
|
||||
|
||||
const result = verifier.verify({ type: 'test' }, null, {});
|
||||
expect(result.decision).toBe('BLOCK');
|
||||
});
|
||||
|
||||
test('should handle empty context gracefully', () => {
|
||||
const action = { type: 'test' };
|
||||
const reasoning = { explanation: 'test' };
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.decision).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Detailed Failure Analysis', () => {
|
||||
test('should provide detailed analysis for failed verifications', () => {
|
||||
const action = {
|
||||
type: 'risky_operation'
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'unclear reasoning'
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
expect(result.analysis).toBeDefined();
|
||||
expect(result.analysis.failed_checks).toBeDefined();
|
||||
expect(result.analysis.recommendations).toBeDefined();
|
||||
});
|
||||
|
||||
test('should suggest improvements for low-confidence actions', () => {
|
||||
const action = {
|
||||
type: 'moderate_operation'
|
||||
};
|
||||
|
||||
const reasoning = {
|
||||
explanation: 'Basic explanation',
|
||||
evidence: ['one piece of evidence']
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, {});
|
||||
|
||||
if (result.confidence < 0.7) {
|
||||
expect(result.suggestions).toBeDefined();
|
||||
expect(result.suggestions.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Singleton Pattern', () => {
|
||||
test('should export singleton instance with required methods', () => {
|
||||
expect(typeof verifier.verify).toBe('function');
|
||||
expect(typeof verifier.getStats).toBe('function');
|
||||
});
|
||||
|
||||
test('should maintain verification history across calls', () => {
|
||||
verifier.verify({ type: 'test' }, { explanation: 'test' }, {});
|
||||
|
||||
const stats = verifier.getStats();
|
||||
|
||||
expect(stats.total_verifications).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Statistics Tracking', () => {
|
||||
test('should track verification statistics', () => {
|
||||
const stats = verifier.getStats();
|
||||
|
||||
expect(stats).toHaveProperty('total_verifications');
|
||||
expect(stats).toHaveProperty('by_decision');
|
||||
expect(stats).toHaveProperty('average_confidence');
|
||||
});
|
||||
|
||||
test('should increment verification count after verify()', () => {
|
||||
const before = verifier.getStats().total_verifications;
|
||||
|
||||
verifier.verify(
|
||||
{ type: 'test' },
|
||||
{ explanation: 'test' },
|
||||
{}
|
||||
);
|
||||
|
||||
const after = verifier.getStats().total_verifications;
|
||||
|
||||
expect(after).toBe(before + 1);
|
||||
});
|
||||
|
||||
test('should track decision distribution', () => {
|
||||
verifier.verify(
|
||||
{ type: 'safe', parameters: {} },
|
||||
{ explanation: 'safe', evidence: ['good evidence'], steps: ['step 1'], alternatives_considered: ['alt'] },
|
||||
{}
|
||||
);
|
||||
|
||||
verifier.verify(
|
||||
{ type: 'unsafe' },
|
||||
{ explanation: 'unclear' },
|
||||
{}
|
||||
);
|
||||
|
||||
const stats = verifier.getStats();
|
||||
|
||||
expect(stats.by_decision.PROCEED + stats.by_decision.BLOCK + stats.by_decision.REQUEST_CONFIRMATION + stats.by_decision.REQUEST_CLARIFICATION).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should calculate average confidence over time', () => {
|
||||
verifier.verify({ type: 'test1' }, { explanation: 'good', evidence: ['a', 'b'], steps: ['1'], alternatives_considered: ['x'] }, {});
|
||||
verifier.verify({ type: 'test2' }, { explanation: 'poor' }, {});
|
||||
|
||||
const stats = verifier.getStats();
|
||||
|
||||
expect(stats.average_confidence).toBeGreaterThan(0);
|
||||
expect(stats.average_confidence).toBeLessThan(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Reasoning Quality Metrics', () => {
|
||||
test('should score high-quality reasoning highly', () => {
|
||||
const reasoning = {
|
||||
explanation: 'Detailed explanation with clear reasoning about why this action is needed and how it aligns with user intent',
|
||||
evidence: [
|
||||
'User explicitly requested this action',
|
||||
'Documentation supports this approach',
|
||||
'Previous similar actions succeeded'
|
||||
],
|
||||
steps: [
|
||||
'Validate preconditions',
|
||||
'Execute action',
|
||||
'Verify results',
|
||||
'Report completion'
|
||||
],
|
||||
alternatives_considered: [
|
||||
'Alternative A: rejected because X',
|
||||
'Alternative B: rejected because Y',
|
||||
'Chosen approach: best because Z'
|
||||
]
|
||||
};
|
||||
|
||||
const score = verifier._assessReasoningQuality(reasoning);
|
||||
|
||||
expect(score).toBeGreaterThan(0.8);
|
||||
});
|
||||
|
||||
test('should score low-quality reasoning poorly', () => {
|
||||
const reasoning = {
|
||||
explanation: 'Do it',
|
||||
evidence: [],
|
||||
steps: []
|
||||
};
|
||||
|
||||
const score = verifier._assessReasoningQuality(reasoning);
|
||||
|
||||
expect(score).toBeLessThan(0.3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Context-Aware Verification', () => {
|
||||
test('should consider recent errors in verification', () => {
|
||||
const action = { type: 'database_operation' };
|
||||
const reasoning = { explanation: 'database op' };
|
||||
|
||||
const errorContext = {
|
||||
errors_recent: 5,
|
||||
last_error_type: 'database_connection'
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, errorContext);
|
||||
|
||||
// Should be more cautious after errors
|
||||
expect(result.confidence_adjustment).toBeLessThan(1.0);
|
||||
});
|
||||
|
||||
test('should consider conversation length in verification', () => {
|
||||
const action = { type: 'operation' };
|
||||
const reasoning = { explanation: 'do operation' };
|
||||
|
||||
const longConversation = {
|
||||
conversation_length: 100
|
||||
};
|
||||
|
||||
const result = verifier.verify(action, reasoning, longConversation);
|
||||
|
||||
// Long conversations should increase scrutiny
|
||||
expect(result.confidence_adjustment).toBeLessThan(1.0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue