fix: refactor MetacognitiveVerifier check methods to return structured objects

MetacognitiveVerifier improvements (48.8% → 56.1% pass rate):

1. Refactored All Check Methods to Return Objects
   - _checkAlignment(): Returns {score, issues[]}
   - _checkCoherence(): Returns {score, issues[]}
   - _checkCompleteness(): Returns {score, missing[]}
   - _checkSafety(): Returns {score, riskLevel, concerns[]}
   - _checkAlternatives(): Returns {score, issues[]}

2. Updated Helper Methods for Backward Compatibility
   - _calculateConfidence(): Handles both object {score: X} and legacy number formats
   - _checkCriticalFailures(): Extracts .score from objects or uses legacy numbers

3. Enhanced Diagnostic Information
   - Alignment: Tracks specific conflicts with instructions
   - Coherence: Identifies missing steps and logical inconsistencies
   - Completeness: Lists unaddressed requirements, missing error handling
   - Safety: Categorizes risk levels (LOW/MEDIUM/CRITICAL), lists concerns
   - Alternatives: Notes missing exploration and rationale

Test Results:
- MetacognitiveVerifier: 23/41 passing (56.1%, +7.3%)
- Overall: 108/192 (56.25%, +3 tests from 105/192)

The structured return values provide detailed context for test assertions
and enable richer verification feedback in production use.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TheFlow 2025-10-07 08:33:29 +13:00
parent 51e10b11ba
commit ecb55994b3

View file

@ -233,6 +233,7 @@ class MetacognitiveVerifier {
_checkAlignment(action, reasoning, context) { _checkAlignment(action, reasoning, context) {
let score = 0.5; // Base score let score = 0.5; // Base score
const issues = [];
// Check cross-reference validation // Check cross-reference validation
const validation = this.validator.validate(action, context); const validation = this.validator.validate(action, context);
@ -240,8 +241,10 @@ class MetacognitiveVerifier {
score += 0.3; score += 0.3;
} else if (validation.status === 'WARNING') { } else if (validation.status === 'WARNING') {
score += 0.1; score += 0.1;
issues.push('Minor conflict detected with user instructions');
} else if (validation.status === 'REJECTED') { } else if (validation.status === 'REJECTED') {
score -= 0.3; score -= 0.3;
issues.push('Action conflicts with explicit user instructions');
} }
// Check if action addresses stated user goal // Check if action addresses stated user goal
@ -256,17 +259,25 @@ class MetacognitiveVerifier {
context.recentUserStatements context.recentUserStatements
); );
score += consistencyScore * 0.2; score += consistencyScore * 0.2;
if (consistencyScore < 0.5) {
issues.push('Action may not be consistent with recent user statements');
}
} }
return Math.min(1.0, Math.max(0.0, score)); return {
score: Math.min(1.0, Math.max(0.0, score)),
issues
};
} }
_checkCoherence(action, reasoning, context) { _checkCoherence(action, reasoning, context) {
let score = 0.7; // Default to reasonable coherence let score = 0.7; // Default to reasonable coherence
const issues = [];
// Check if reasoning steps are provided // Check if reasoning steps are provided
if (!reasoning.steps || reasoning.steps.length === 0) { if (!reasoning.steps || reasoning.steps.length === 0) {
score -= 0.2; score -= 0.2;
issues.push('No reasoning steps provided');
} }
// Check for logical consistency // Check for logical consistency
@ -279,48 +290,67 @@ class MetacognitiveVerifier {
score += 0.2; score += 0.2;
} else { } else {
score -= 0.3; score -= 0.3;
issues.push('Logical inconsistency detected between assumptions and conclusions');
} }
} }
// Check for internal contradictions // Check for internal contradictions
if (this._hasContradictions(reasoning)) { if (this._hasContradictions(reasoning)) {
score -= 0.4; score -= 0.4;
issues.push('Internal contradictions detected in reasoning');
} }
return Math.min(1.0, Math.max(0.0, score)); return {
score: Math.min(1.0, Math.max(0.0, score)),
issues
};
} }
_checkCompleteness(action, reasoning, context) { _checkCompleteness(action, reasoning, context) {
let score = 0.6; // Base score let score = 0.6; // Base score
const missing = [];
// Check if all stated requirements are addressed // Check if all stated requirements are addressed
if (context.requirements) { if (context.requirements) {
const addressedCount = context.requirements.filter(req => const unaddressed = context.requirements.filter(req =>
this._isRequirementAddressed(req, action, reasoning) !this._isRequirementAddressed(req, action, reasoning)
).length; );
const addressedCount = context.requirements.length - unaddressed.length;
score += (addressedCount / context.requirements.length) * 0.3; score += (addressedCount / context.requirements.length) * 0.3;
unaddressed.forEach(req => missing.push(`Requirement not addressed: ${req}`));
} }
// Check for edge cases consideration // Check for edge cases consideration
if (reasoning.edgeCases && reasoning.edgeCases.length > 0) { if (reasoning.edgeCases && reasoning.edgeCases.length > 0) {
score += 0.1; score += 0.1;
} else {
missing.push('No edge cases considered');
} }
// Check for error handling // Check for error handling
if (reasoning.errorHandling || action.errorHandling) { if (reasoning.errorHandling || action.errorHandling) {
score += 0.1; score += 0.1;
} else {
missing.push('No error handling specified');
} }
return Math.min(1.0, Math.max(0.0, score)); return {
score: Math.min(1.0, Math.max(0.0, score)),
missing
};
} }
_checkSafety(action, reasoning, context) { _checkSafety(action, reasoning, context) {
let score = 0.8; // Default to safe unless red flags let score = 0.8; // Default to safe unless red flags
const concerns = [];
let riskLevel = 'LOW';
// Check boundary enforcement // Check boundary enforcement
const boundaryCheck = this.enforcer.enforce(action, context); const boundaryCheck = this.enforcer.enforce(action, context);
if (!boundaryCheck.allowed) { if (!boundaryCheck.allowed) {
score -= 0.5; // Major safety concern score -= 0.5; // Major safety concern
concerns.push('Action violates Tractatus boundaries');
riskLevel = 'CRITICAL';
} }
// Check for destructive operations // Check for destructive operations
@ -334,6 +364,8 @@ class MetacognitiveVerifier {
for (const pattern of destructivePatterns) { for (const pattern of destructivePatterns) {
if (pattern.test(actionText)) { if (pattern.test(actionText)) {
score -= 0.2; score -= 0.2;
concerns.push('Destructive operation detected');
riskLevel = riskLevel === 'LOW' ? 'MEDIUM' : riskLevel;
break; break;
} }
} }
@ -341,35 +373,50 @@ class MetacognitiveVerifier {
// Check if data backup is mentioned for risky operations // Check if data backup is mentioned for risky operations
if (score < 0.7 && !reasoning.backupMentioned) { if (score < 0.7 && !reasoning.backupMentioned) {
score -= 0.1; score -= 0.1;
concerns.push('No backup mentioned for risky operation');
} }
// Check for validation before execution // Check for validation before execution
if (action.requiresValidation && !reasoning.validationPlanned) { if (action.requiresValidation && !reasoning.validationPlanned) {
score -= 0.1; score -= 0.1;
concerns.push('No validation planned before execution');
} }
return Math.min(1.0, Math.max(0.0, score)); return {
score: Math.min(1.0, Math.max(0.0, score)),
riskLevel,
concerns
};
} }
_checkAlternatives(action, reasoning, context) { _checkAlternatives(action, reasoning, context) {
let score = 0.5; // Base score let score = 0.5; // Base score
const issues = [];
// Check if alternatives were considered // Check if alternatives were considered
if (reasoning.alternativesConsidered && reasoning.alternativesConsidered.length > 0) { if (reasoning.alternativesConsidered && reasoning.alternativesConsidered.length > 0) {
score += 0.3; score += 0.3;
} else {
issues.push('No alternatives considered');
} }
// Check if rationale for chosen approach is provided // Check if rationale for chosen approach is provided
if (reasoning.chosenBecause) { if (reasoning.chosenBecause) {
score += 0.2; score += 0.2;
} else {
issues.push('No rationale provided for chosen approach');
} }
// Lower score if action seems like first idea without exploration // Lower score if action seems like first idea without exploration
if (!reasoning.alternativesConsidered && !reasoning.explored) { if (!reasoning.alternativesConsidered && !reasoning.explored) {
score -= 0.2; score -= 0.2;
issues.push('Appears to be first idea without exploration');
} }
return Math.min(1.0, Math.max(0.0, score)); return {
score: Math.min(1.0, Math.max(0.0, score)),
issues
};
} }
_calculateConfidence(scores) { _calculateConfidence(scores) {
@ -377,7 +424,9 @@ class MetacognitiveVerifier {
for (const [dimension, dimensionConfig] of Object.entries(this.dimensions)) { for (const [dimension, dimensionConfig] of Object.entries(this.dimensions)) {
const key = dimension.toLowerCase(); const key = dimension.toLowerCase();
const score = scores[key] || 0.5; const scoreData = scores[key];
// Handle both object format {score: X} and legacy number format
const score = typeof scoreData === 'object' ? (scoreData.score || 0.5) : (scoreData || 0.5);
confidence += score * dimensionConfig.weight; confidence += score * dimensionConfig.weight;
} }
@ -402,7 +451,9 @@ class MetacognitiveVerifier {
for (const [dimension, config] of Object.entries(this.dimensions)) { for (const [dimension, config] of Object.entries(this.dimensions)) {
const key = dimension.toLowerCase(); const key = dimension.toLowerCase();
const score = scores[key]; const scoreData = scores[key];
// Handle both object format {score: X} and legacy number format
const score = typeof scoreData === 'object' ? (scoreData.score || 0.5) : (scoreData || 0.5);
if (score < config.criticalThreshold) { if (score < config.criticalThreshold) {
failures.push({ failures.push({