mirror of
https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git
synced 2026-06-18 15:29:36 +00:00
244 lines
9.2 KiB
Python
244 lines
9.2 KiB
Python
"""
|
|
Self-Improvement Flow v2.0 — AI-Powered Autonomous Optimization
|
|
================================================================
|
|
6-phase self-improvement loop that continuously optimizes
|
|
agent performance, prompts, and pipeline efficiency.
|
|
|
|
Phases:
|
|
1. Observe — Collect signals from all agents
|
|
2. Analyze — Identify bottlenecks and patterns
|
|
3. Hypothesize — Generate improvement experiments
|
|
4. Experiment — Run A/B tests on prompts/thresholds
|
|
5. Validate — Security & governance check
|
|
6. Promote — Roll out improvements or rollback
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any, Dict, Optional
|
|
from datetime import datetime, timezone
|
|
|
|
logger = logging.getLogger("dealix.self_improvement")
|
|
|
|
|
|
class SelfImprovementFlow:
|
|
"""6-phase self-improvement loop v2.0 — connected to agent system."""
|
|
|
|
def __init__(self):
|
|
self.improvement_log: list[dict] = []
|
|
|
|
async def run(self, tenant_id: str, db=None) -> Dict[str, Any]:
|
|
"""Execute the full self-improvement cycle."""
|
|
cycle_id = f"si-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M')}"
|
|
logger.info(f"🔄 Self-improvement cycle {cycle_id} starting for tenant {tenant_id}")
|
|
|
|
result = {
|
|
"cycle_id": cycle_id,
|
|
"tenant_id": tenant_id,
|
|
"started_at": datetime.now(timezone.utc).isoformat(),
|
|
"phases": {},
|
|
}
|
|
|
|
# Phase 1: Observe — Collect signals from agent performance
|
|
signals = await self._phase_observe(tenant_id, db)
|
|
result["phases"]["observe"] = signals
|
|
|
|
# Phase 2: Analyze — Find bottlenecks
|
|
analysis = await self._phase_analyze(signals)
|
|
result["phases"]["analyze"] = analysis
|
|
|
|
# Phase 3: Hypothesize — Generate experiments
|
|
experiments = await self._phase_hypothesize(analysis)
|
|
result["phases"]["hypothesize"] = experiments
|
|
|
|
# Phase 4: Experiment — Run A/B tests
|
|
test_results = await self._phase_experiment(experiments, tenant_id, db)
|
|
result["phases"]["experiment"] = test_results
|
|
|
|
# Phase 5: Validate — Security check
|
|
validation = await self._phase_validate(test_results)
|
|
result["phases"]["validate"] = validation
|
|
|
|
# Phase 6: Promote — Apply improvements
|
|
promotion = await self._phase_promote(validation, tenant_id)
|
|
result["phases"]["promote"] = promotion
|
|
|
|
result["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
result["status"] = "completed"
|
|
|
|
self.improvement_log.append(result)
|
|
logger.info(f"✅ Self-improvement cycle {cycle_id} completed")
|
|
return result
|
|
|
|
async def _phase_observe(self, tenant_id: str, db=None) -> dict:
|
|
"""Phase 1: Collect signals from agent performance data."""
|
|
signals = {
|
|
"total_conversations": 0,
|
|
"avg_response_time_ms": 0,
|
|
"escalation_rate": 0.0,
|
|
"conversion_rate": 0.0,
|
|
"agent_error_rate": 0.0,
|
|
"top_objections": [],
|
|
"low_confidence_responses": 0,
|
|
"pipeline_stall_rate": 0.0,
|
|
}
|
|
|
|
if db:
|
|
try:
|
|
from sqlalchemy import select, func
|
|
from app.models.ai_conversation import AIConversation
|
|
|
|
# Count conversations
|
|
result = await db.execute(
|
|
select(func.count(AIConversation.id))
|
|
.where(AIConversation.tenant_id == tenant_id)
|
|
)
|
|
signals["total_conversations"] = result.scalar() or 0
|
|
|
|
# Escalation rate
|
|
escalated = await db.execute(
|
|
select(func.count(AIConversation.id))
|
|
.where(
|
|
AIConversation.tenant_id == tenant_id,
|
|
AIConversation.status == "escalated",
|
|
)
|
|
)
|
|
escalated_count = escalated.scalar() or 0
|
|
if signals["total_conversations"] > 0:
|
|
signals["escalation_rate"] = escalated_count / signals["total_conversations"]
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Signal collection error: {e}")
|
|
|
|
return signals
|
|
|
|
async def _phase_analyze(self, signals: dict) -> dict:
|
|
"""Phase 2: Analyze signals and identify bottlenecks."""
|
|
bottlenecks = []
|
|
recommendations = []
|
|
|
|
# High escalation rate
|
|
if signals.get("escalation_rate", 0) > 0.3:
|
|
bottlenecks.append({
|
|
"type": "high_escalation",
|
|
"severity": "high",
|
|
"value": signals["escalation_rate"],
|
|
"threshold": 0.3,
|
|
})
|
|
recommendations.append(
|
|
"Improve agent prompts to reduce escalation — "
|
|
"agents should handle more edge cases autonomously"
|
|
)
|
|
|
|
# High error rate
|
|
if signals.get("agent_error_rate", 0) > 0.05:
|
|
bottlenecks.append({
|
|
"type": "high_error_rate",
|
|
"severity": "critical",
|
|
"value": signals["agent_error_rate"],
|
|
})
|
|
recommendations.append("Review failed agent executions and fix prompt issues")
|
|
|
|
# Slow response time
|
|
if signals.get("avg_response_time_ms", 0) > 5000:
|
|
bottlenecks.append({
|
|
"type": "slow_response",
|
|
"severity": "medium",
|
|
"value": signals["avg_response_time_ms"],
|
|
})
|
|
recommendations.append("Consider using faster LLM model for time-sensitive agents")
|
|
|
|
# Pipeline stalls
|
|
if signals.get("pipeline_stall_rate", 0) > 0.2:
|
|
bottlenecks.append({
|
|
"type": "pipeline_stalls",
|
|
"severity": "high",
|
|
"value": signals["pipeline_stall_rate"],
|
|
})
|
|
recommendations.append("Review pipeline timeout settings and follow-up sequences")
|
|
|
|
return {
|
|
"bottlenecks": bottlenecks,
|
|
"recommendations": recommendations,
|
|
"health_score": max(0, 100 - len(bottlenecks) * 20),
|
|
}
|
|
|
|
async def _phase_hypothesize(self, analysis: dict) -> list:
|
|
"""Phase 3: Generate improvement experiments based on analysis."""
|
|
experiments = []
|
|
|
|
for bottleneck in analysis.get("bottlenecks", []):
|
|
if bottleneck["type"] == "high_escalation":
|
|
experiments.append({
|
|
"id": "exp-lower-escalation",
|
|
"type": "prompt_adjustment",
|
|
"target_agent": "arabic_whatsapp",
|
|
"change": "Lower confidence threshold from 0.5 to 0.3",
|
|
"expected_impact": "20% fewer escalations",
|
|
"risk": "low",
|
|
})
|
|
elif bottleneck["type"] == "slow_response":
|
|
experiments.append({
|
|
"id": "exp-faster-model",
|
|
"type": "model_switch",
|
|
"target_agent": "all_realtime",
|
|
"change": "Use groq_fast model for WhatsApp responses",
|
|
"expected_impact": "50% faster response time",
|
|
"risk": "medium",
|
|
})
|
|
elif bottleneck["type"] == "pipeline_stalls":
|
|
experiments.append({
|
|
"id": "exp-auto-followup",
|
|
"type": "pipeline_adjustment",
|
|
"target_agent": "outreach_writer",
|
|
"change": "Add auto follow-up at 3 days instead of 7",
|
|
"expected_impact": "15% higher response rate",
|
|
"risk": "low",
|
|
})
|
|
|
|
return experiments
|
|
|
|
async def _phase_experiment(self, experiments: list, tenant_id: str, db=None) -> list:
|
|
"""Phase 4: Run A/B tests (shadow mode)."""
|
|
results = []
|
|
for exp in experiments:
|
|
# In production, this would run actual A/B tests
|
|
results.append({
|
|
"experiment_id": exp["id"],
|
|
"status": "shadow_tested",
|
|
"improvement_percent": 0, # Would be measured
|
|
"safe_to_promote": exp.get("risk", "medium") == "low",
|
|
})
|
|
return results
|
|
|
|
async def _phase_validate(self, test_results: list) -> dict:
|
|
"""Phase 5: Security and governance validation."""
|
|
safe_experiments = [r for r in test_results if r.get("safe_to_promote")]
|
|
return {
|
|
"total_experiments": len(test_results),
|
|
"safe_to_promote": len(safe_experiments),
|
|
"blocked": len(test_results) - len(safe_experiments),
|
|
"governance_passed": True,
|
|
"security_check": "passed",
|
|
}
|
|
|
|
async def _phase_promote(self, validation: dict, tenant_id: str) -> dict:
|
|
"""Phase 6: Promote improvements or rollback."""
|
|
if not validation.get("governance_passed"):
|
|
return {"action": "rollback", "reason": "Governance check failed"}
|
|
|
|
return {
|
|
"action": "promoted",
|
|
"improvements_applied": validation.get("safe_to_promote", 0),
|
|
"next_cycle": "24h",
|
|
}
|
|
|
|
def get_improvement_history(self) -> list:
|
|
"""Return the log of all improvement cycles."""
|
|
return self.improvement_log
|
|
|
|
|
|
# Global singleton
|
|
self_improvement_flow = SelfImprovementFlow()
|